GNUmakefile

@@ -68,6 +68,7 @@ BLOSC_CPPFLAGS := $(BLOSC_CPPFLAGS) \
 	-I$(VELOC_INSTALL_DIR)/include
 BASE_CXXFLAGS := \
 	-L$(VELOC_INSTALL_DIR)/lib -L$(VELOC_INSTALL_DIR)/lib64 \
+	-Wl,-rpath,$(VELOC_INSTALL_DIR)/lib64 -Wl,-rpath,$(VELOC_INSTALL_DIR)/lib\
 	-Wl,--copy-dt-needed-entries -lveloc-client
 endif
 

GenericIO.h

@@ -461,9 +461,25 @@ public:
 
   static void setNaturalDefaultPartition();
 
-  static void flushAll() {
+  static bool flushAll() {
+      bool ret = true;
+#ifdef GENERICIO_WITH_VELOC
+      bool retLocal = GenericFileIO_VELOC::flush();//With veloc do a blocking flush of anything outstanding and return if all outstanding tasks were successful
+#ifndef GENERICIO_NO_MPI
+      MPI_Allreduce(&retLocal, &ret, 1, MPI_C_BOOL, MPI_LAND, MPI_COMM_WORLD);//Global reduction on if flush was successful 
+#else
+      ret = retLocal;
+#endif
+#endif//GENERICIO_WITH_VELOC
+      return ret;//by default return that flush was successful (as it does not do anything)
+  }
+
+  static void shutdown() {
 #ifdef GENERICIO_WITH_VELOC
       GenericFileIO_VELOC::flush();
+#ifndef GENERICIO_NO_MPI
+      MPI_Barrier(MPI_COMM_WORLD);//Barrier to wait for all ranks to finish
+#endif
 #endif
   }
 

GenericIOBenchmarkWrite.cxx

@@ -128,7 +128,7 @@ int main(int argc, char *argv[]) {
   GenericIO::setNaturalDefaultPartition();
 
   char *mpiioName = argv[a++];
-  size_t Np = atoi(argv[a++])/commRanks;
+  size_t Np = atol(argv[a++])/commRanks;
   int seed = atoi(argv[a++]);
 
   srand48(seed + commRank);

README.md

@@ -115,10 +115,11 @@ Instructions can be found here: [https://veloc.readthedocs.io](https://veloc.rea
 
 **Install**
 
-Set the VELOC_INSTALL_DIR variable in GNUMakefile to the root of the VELOC installation directory. 
+Set the VELOC_INSTALL_DIR variable in GNUMakefile to the root of the VELOC installation directory.
 Then proceed to compile and link GIO as usual.
 
 **Run**
 
-Define the GENERICIO_USE_VELOC environment variable as the path to the VELOC configuration file.
-An example is available here: veloc.cfg
+Define the GENERICIO_USE_VELOC environment variable as the path to the scratch directory.
+The scratch directory will be used as a local cache and needs to be a NVMe mount point on the compute node.
+Define the VELOC_MAX_CACHE_SIZE environment variable as the maximum size (in bytes) of unflushed data allowed in the scratch folder.

thirdparty/veloc/FileIOVELOC.cxx

@@ -40,8 +40,11 @@ void GenericFileIO_VELOC::write(const void *buf, size_t count, off_t offset, con
 	throw std::runtime_error("Unable to write " + D + " to file: " + FileName + ": " + strerror(errno));
 }
 
-void GenericFileIO_VELOC::flush() {
-    veloc::cached_file_t::flush();
+bool GenericFileIO_VELOC::flush() {
+    return veloc::cached_file_t::flush();
+}
+
+void GenericFileIO_VELOC::shutdown() {
     veloc::cached_file_t::shutdown();
 }
 

thirdparty/veloc/FileIOVELOC.hpp

@@ -15,7 +15,8 @@ public:
     void setSize(size_t sz);
     void read(void *buf, size_t count, off_t offset, const std::string &D);
     void write(const void *buf, size_t count, off_t offset, const std::string &D);
-    static void flush();
+    static bool flush();
+    static void shutdown();
 
 protected:
     int FH;

veloc.cfg

-scratch = /tmp/scratch
-persistent = /tmp/persistent
-mode = async
\ No newline at end of file