diff --git a/GenericIO.cxx b/GenericIO.cxx index 1912a48b4af20e6c44a6f18d66a74af2122e843d..4ab1cb31beb6c0296f87665ecf7f3196e89a3d5a 100644 --- a/GenericIO.cxx +++ b/GenericIO.cxx @@ -53,6 +53,7 @@ extern "C" { #include <stdexcept> #include <iterator> #include <algorithm> +#include <set> #include <cassert> #include <cstddef> #include <cstring> @@ -394,6 +395,17 @@ static int GetSZDT(GenericIO::Variable &Var) { return -1; } +void GenericIO::setFH(int R) { +#ifndef GENERICIO_NO_MPI + if (FileIOType == FileIOMPI) + FH.get() = new GenericFileIO_MPI(R); + else if (FileIOType == FileIOMPICollective) + FH.get() = new GenericFileIO_MPICollective(R); + else +#endif + FH.get() = new GenericFileIO_POSIX(); +} + #ifndef GENERICIO_NO_MPI void GenericIO::write() { if (isBigEndian()) @@ -423,6 +435,13 @@ void GenericIO::write() { MPI_Comm_rank(SplitComm, &SplitRank); MPI_Comm_size(SplitComm, &SplitNRanks); + bool Rank0CreateAll = false; + const char *EnvStr = getenv("GENERICIO_RANK0_CREATE_ALL"); + if (EnvStr) { + int Mod = atoi(EnvStr); + Rank0CreateAll = (Mod > 0); + } + string LocalFileName; if (SplitNRanks != NRanks) { if (Rank == 0) { @@ -466,6 +485,24 @@ void GenericIO::write() { } GIO.write(); + + // On some file systems, it can be very expensive for multiple ranks to + // create files in the same directory. Creating a new file requires a + // kind of exclusive lock that is expensive to obtain. + if (Rank0CreateAll) { + set<int> AllPartitions; + for (int i = 0; i < NRanks; ++i) AllPartitions.insert(MapPartition[i]); + + for (set<int>::iterator i = AllPartitions.begin(), + e = AllPartitions.end(); i != e; ++i) { + stringstream ss; + ss << FileName << "#" << *i; + + setFH(MPI_COMM_SELF); + FH.get()->open(ss.str()); + close(); + } + } } else { MPI_Gather(&Partition, 1, MPI_INT, 0, 0, MPI_INT, 0, Comm); } @@ -498,7 +535,7 @@ void GenericIO::write() { RHLocal.GlobalRank = Rank; bool ShouldCompress = DefaultShouldCompress; - const char *EnvStr = getenv("GENERICIO_COMPRESS"); + EnvStr = getenv("GENERICIO_COMPRESS"); if (EnvStr) { int Mod = atoi(EnvStr); ShouldCompress = (Mod > 0); @@ -754,14 +791,9 @@ nocomp: uint64_t HeaderCRC = crc64_omp(&Header[0], HeaderSize - CRCSize); crc64_invert(HeaderCRC, &Header[HeaderSize - CRCSize]); - if (FileIOType == FileIOMPI) - FH.get() = new GenericFileIO_MPI(MPI_COMM_SELF); - else if (FileIOType == FileIOMPICollective) - FH.get() = new GenericFileIO_MPICollective(MPI_COMM_SELF); - else - FH.get() = new GenericFileIO_POSIX(); + setFH(MPI_COMM_SELF); - FH.get()->open(LocalFileName); + FH.get()->open(LocalFileName, false, Rank0CreateAll); FH.get()->setSize(FileSize); FH.get()->write(&Header[0], HeaderSize, 0, "header"); @@ -779,12 +811,7 @@ nocomp: MPI_Barrier(SplitComm); - if (FileIOType == FileIOMPI) - FH.get() = new GenericFileIO_MPI(SplitComm); - else if (FileIOType == FileIOMPICollective) - FH.get() = new GenericFileIO_MPICollective(SplitComm); - else - FH.get() = new GenericFileIO_POSIX(); + setFH(SplitComm); FH.get()->open(LocalFileName, false, true); @@ -1021,14 +1048,13 @@ void GenericIO::openAndReadHeader(MismatchBehavior MB, int EffRank, bool CheckPa vector<char> Header; if (SplitRank == 0) { + setFH( #ifndef GENERICIO_NO_MPI - if (FileIOType == FileIOMPI) - FH.get() = new GenericFileIO_MPI(MPI_COMM_SELF); - else if (FileIOType == FileIOMPICollective) - FH.get() = new GenericFileIO_MPICollective(MPI_COMM_SELF); - else + MPI_COMM_SELF +#else + 0 #endif - FH.get() = new GenericFileIO_POSIX(); + ); #ifndef GENERICIO_NO_MPI char True = 1, False = 0; @@ -1092,12 +1118,7 @@ void GenericIO::openAndReadHeader(MismatchBehavior MB, int EffRank, bool CheckPa if (!DisableCollErrChecking) MPI_Barrier(Comm); - if (FileIOType == FileIOMPI) - FH.get() = new GenericFileIO_MPI(SplitComm); - else if (FileIOType == FileIOMPICollective) - FH.get() = new GenericFileIO_MPICollective(SplitComm); - else - FH.get() = new GenericFileIO_POSIX(); + setFH(SplitComm); int OpenErr = 0, TotOpenErr; try { diff --git a/GenericIO.h b/GenericIO.h index cfb37b16ddd3b5ccea91b2d5b55392c56a8740cb..f4f471b719e2f478cec1e87a22f3d51a308a563e 100644 --- a/GenericIO.h +++ b/GenericIO.h @@ -637,6 +637,8 @@ public: FHWCnt *CountedFH; } FH; + + void setFH(int R); }; } /* END namespace cosmotk */