From 05d6efc74c9ea457f9e5a426214ee9252b426cd9 Mon Sep 17 00:00:00 2001
From: Michael Buehlmann <buehlmann.michi@gmail.com>
Date: Fri, 3 Mar 2023 17:22:59 -0600
Subject: [PATCH] change default genericio file partitions
---
.gitignore | 3 ++-
GenericIO.cxx | 33 +++++++++++++++++++++++++--------
2 files changed, 27 insertions(+), 9 deletions(-)
diff --git a/.gitignore b/.gitignore
index d5a3234..516768e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,4 +5,5 @@ frontend/
*.pyc
*.egg-info
.vscode
-docs/_build
\ No newline at end of file
+docs/_build
+.cache
\ No newline at end of file
diff --git a/GenericIO.cxx b/GenericIO.cxx
index 77d8380..ac00f34 100644
--- a/GenericIO.cxx
+++ b/GenericIO.cxx
@@ -1867,7 +1867,13 @@ void GenericIO::setNaturalDefaultPartition() {
#ifdef __bgq__
DefaultPartition = MPIX_IO_link_id();
#elif !defined(GENERICIO_NO_MPI)
- bool UseName = true;
+
+ int rank, nranks;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &nranks);
+
+ // assign by 8bit hash of MPI_Get_processor_name
+ bool UseName = false;
const char *EnvStr = getenv("GENERICIO_PARTITIONS_USE_NAME");
if (EnvStr) {
int Mod = atoi(EnvStr);
@@ -1888,15 +1894,26 @@ void GenericIO::setNaturalDefaultPartition() {
DefaultPartition = color;
}
- // This is for debugging.
+ // assign round robin to max number given by GENERICIO_RANK_PARTITIONS
+ int roundRobinAssign = 0;
EnvStr = getenv("GENERICIO_RANK_PARTITIONS");
if (EnvStr) {
- int Mod = atoi(EnvStr);
- if (Mod > 0) {
- int Rank;
- MPI_Comm_rank(MPI_COMM_WORLD, &Rank);
- DefaultPartition += Rank % Mod;
- }
+ roundRobinAssign = atoi(EnvStr);
+ }
+
+ // default: one file per rank, up to 256 files. Warn if we combine ranks in
+ // single file
+ if(!UseName && (roundRobinAssign==0)) {
+ if(rank == 0 && nranks > 256)
+ std::cerr << "WARNING: Running with more than 256 MPI ranks, and "
+ << "GENERICIO_RANK_PARTITIONS is not set.\n"
+ << "GenericIO will limit the number of file partitions to 256"
+ << std::endl;
+ roundRobinAssign = 256;
+ }
+
+ if (roundRobinAssign > 0) {
+ DefaultPartition = rank % roundRobinAssign;
}
#endif
#ifdef GENERICIO_WITH_VELOC
--
GitLab