From f66a4a0ff9b3928ffb77a95f70cf9a449e6da3e7 Mon Sep 17 00:00:00 2001
From: Hal Finkel <hfinkel@anl.gov>
Date: Thu, 30 Apr 2020 22:35:07 -0500
Subject: [PATCH] Add an option to have rank 0 create all files and subfiles
---
GenericIO.cxx | 73 +++++++++++++++++++++++++++++++++------------------
GenericIO.h | 2 ++
2 files changed, 49 insertions(+), 26 deletions(-)
diff --git a/GenericIO.cxx b/GenericIO.cxx
index 1912a48..4ab1cb3 100644
--- a/GenericIO.cxx
+++ b/GenericIO.cxx
@@ -53,6 +53,7 @@ extern "C" {
#include <stdexcept>
#include <iterator>
#include <algorithm>
+#include <set>
#include <cassert>
#include <cstddef>
#include <cstring>
@@ -394,6 +395,17 @@ static int GetSZDT(GenericIO::Variable &Var) {
return -1;
}
+void GenericIO::setFH(int R) {
+#ifndef GENERICIO_NO_MPI
+ if (FileIOType == FileIOMPI)
+ FH.get() = new GenericFileIO_MPI(R);
+ else if (FileIOType == FileIOMPICollective)
+ FH.get() = new GenericFileIO_MPICollective(R);
+ else
+#endif
+ FH.get() = new GenericFileIO_POSIX();
+}
+
#ifndef GENERICIO_NO_MPI
void GenericIO::write() {
if (isBigEndian())
@@ -423,6 +435,13 @@ void GenericIO::write() {
MPI_Comm_rank(SplitComm, &SplitRank);
MPI_Comm_size(SplitComm, &SplitNRanks);
+ bool Rank0CreateAll = false;
+ const char *EnvStr = getenv("GENERICIO_RANK0_CREATE_ALL");
+ if (EnvStr) {
+ int Mod = atoi(EnvStr);
+ Rank0CreateAll = (Mod > 0);
+ }
+
string LocalFileName;
if (SplitNRanks != NRanks) {
if (Rank == 0) {
@@ -466,6 +485,24 @@ void GenericIO::write() {
}
GIO.write();
+
+ // On some file systems, it can be very expensive for multiple ranks to
+ // create files in the same directory. Creating a new file requires a
+ // kind of exclusive lock that is expensive to obtain.
+ if (Rank0CreateAll) {
+ set<int> AllPartitions;
+ for (int i = 0; i < NRanks; ++i) AllPartitions.insert(MapPartition[i]);
+
+ for (set<int>::iterator i = AllPartitions.begin(),
+ e = AllPartitions.end(); i != e; ++i) {
+ stringstream ss;
+ ss << FileName << "#" << *i;
+
+ setFH(MPI_COMM_SELF);
+ FH.get()->open(ss.str());
+ close();
+ }
+ }
} else {
MPI_Gather(&Partition, 1, MPI_INT, 0, 0, MPI_INT, 0, Comm);
}
@@ -498,7 +535,7 @@ void GenericIO::write() {
RHLocal.GlobalRank = Rank;
bool ShouldCompress = DefaultShouldCompress;
- const char *EnvStr = getenv("GENERICIO_COMPRESS");
+ EnvStr = getenv("GENERICIO_COMPRESS");
if (EnvStr) {
int Mod = atoi(EnvStr);
ShouldCompress = (Mod > 0);
@@ -754,14 +791,9 @@ nocomp:
uint64_t HeaderCRC = crc64_omp(&Header[0], HeaderSize - CRCSize);
crc64_invert(HeaderCRC, &Header[HeaderSize - CRCSize]);
- if (FileIOType == FileIOMPI)
- FH.get() = new GenericFileIO_MPI(MPI_COMM_SELF);
- else if (FileIOType == FileIOMPICollective)
- FH.get() = new GenericFileIO_MPICollective(MPI_COMM_SELF);
- else
- FH.get() = new GenericFileIO_POSIX();
+ setFH(MPI_COMM_SELF);
- FH.get()->open(LocalFileName);
+ FH.get()->open(LocalFileName, false, Rank0CreateAll);
FH.get()->setSize(FileSize);
FH.get()->write(&Header[0], HeaderSize, 0, "header");
@@ -779,12 +811,7 @@ nocomp:
MPI_Barrier(SplitComm);
- if (FileIOType == FileIOMPI)
- FH.get() = new GenericFileIO_MPI(SplitComm);
- else if (FileIOType == FileIOMPICollective)
- FH.get() = new GenericFileIO_MPICollective(SplitComm);
- else
- FH.get() = new GenericFileIO_POSIX();
+ setFH(SplitComm);
FH.get()->open(LocalFileName, false, true);
@@ -1021,14 +1048,13 @@ void GenericIO::openAndReadHeader(MismatchBehavior MB, int EffRank, bool CheckPa
vector<char> Header;
if (SplitRank == 0) {
+ setFH(
#ifndef GENERICIO_NO_MPI
- if (FileIOType == FileIOMPI)
- FH.get() = new GenericFileIO_MPI(MPI_COMM_SELF);
- else if (FileIOType == FileIOMPICollective)
- FH.get() = new GenericFileIO_MPICollective(MPI_COMM_SELF);
- else
+ MPI_COMM_SELF
+#else
+ 0
#endif
- FH.get() = new GenericFileIO_POSIX();
+ );
#ifndef GENERICIO_NO_MPI
char True = 1, False = 0;
@@ -1092,12 +1118,7 @@ void GenericIO::openAndReadHeader(MismatchBehavior MB, int EffRank, bool CheckPa
if (!DisableCollErrChecking)
MPI_Barrier(Comm);
- if (FileIOType == FileIOMPI)
- FH.get() = new GenericFileIO_MPI(SplitComm);
- else if (FileIOType == FileIOMPICollective)
- FH.get() = new GenericFileIO_MPICollective(SplitComm);
- else
- FH.get() = new GenericFileIO_POSIX();
+ setFH(SplitComm);
int OpenErr = 0, TotOpenErr;
try {
diff --git a/GenericIO.h b/GenericIO.h
index cfb37b1..f4f471b 100644
--- a/GenericIO.h
+++ b/GenericIO.h
@@ -637,6 +637,8 @@ public:
FHWCnt *CountedFH;
} FH;
+
+ void setFH(int R);
};
} /* END namespace cosmotk */
--
GitLab