diff --git a/GNUmakefile b/GNUmakefile
index 4881c6e9b857a131ad4f5e3dec7d1ed313134f8a..aea9ccedc1ad1e051d13f21ec20ba0b928494c73 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -60,6 +60,17 @@ BLOSC_CPPFLAGS := \
-Ithirdparty/blosc/internal-complibs/zstd-0.7.4/decompress \
-Ithirdparty/SZ/sz/include
+# Set the VELOC_INSTALL_DIR environment variable to enable VELOC support
+ifdef VELOC_INSTALL_DIR
+BLOSC_CPPFLAGS := $(BLOSC_CPPFLAGS) \
+ -DGENERICIO_WITH_VELOC \
+ -Ithirdparty/veloc \
+ -I$(VELOC_INSTALL_DIR)/include
+BASE_CXXFLAGS := \
+ -L$(VELOC_INSTALL_DIR)/lib -L$(VELOC_INSTALL_DIR)/lib64 \
+ -Wl,--copy-dt-needed-entries -lveloc-client
+endif
+
BASE_CPPFLAGS := $(BLOSC_CPPFLAGS) -I. -D__STDC_CONSTANT_MACROS
UNAME_S := $(shell uname -s)
@@ -75,12 +86,12 @@ endif
FEDIR = frontend
FE_CFLAGS := -g -fPIC -O3 $(OPENMPFLAG) $(G99FLAG)
-FE_CXXFLAGS := -g -fPIC -O3 $(OPENMPFLAG)
+FE_CXXFLAGS := -g -fPIC -O3 $(OPENMPFLAG) $(BASE_CXXFLAGS)
FE_CPPFLAGS := $(BASE_CPPFLAGS) -Ithirdparty/sqlite -DGENERICIO_NO_MPI
MPIDIR = mpi
MPI_CFLAGS := -g -O3 $(OPENMPFLAG) $(G99FLAG)
-MPI_CXXFLAGS := -g -O3 $(OPENMPFLAG)
+MPI_CXXFLAGS := -g -O3 $(OPENMPFLAG) $(BASE_CXXFLAGS)
MPI_CPPFLAGS := $(BASE_CPPFLAGS)
$(FEDIR):
@@ -189,6 +200,11 @@ BLOSC_O := \
thirdparty/SZ/sz/src/szd_double_ts.o \
thirdparty/SZ/sz/src/szd_float_ts.o
+ifdef VELOC_INSTALL_DIR
+BLOSC_O := $(BLOSC_O) \
+ thirdparty/veloc/FileIOVELOC.o
+endif
+
FE_BLOSC_O := $(addprefix $(FEDIR)/,$(BLOSC_O))
$(FEDIR)/GenericIOPrint: $(FEDIR)/GenericIOPrint.o $(FEDIR)/GenericIO.o $(FE_BLOSC_O)
@@ -259,4 +275,3 @@ fe-sqlite: frontend-sqlite
clean:
rm -rf frontend mpi legacy_python/genericio.pyc
-
diff --git a/GenericFileIO.h b/GenericFileIO.h
new file mode 100644
index 0000000000000000000000000000000000000000..33d6fe3fd4790f92c1d80fad0f7937d4f66ba034
--- /dev/null
+++ b/GenericFileIO.h
@@ -0,0 +1,24 @@
+#ifndef __GENERIC_FILE_IO
+#define __GENERIC_FILE_IO
+
+#include <string>
+
+namespace gio {
+class GenericFileIO {
+public:
+ virtual ~GenericFileIO() {}
+
+public:
+ virtual void open(const std::string &FN, bool ForReading = false, bool MustExist = false) = 0;
+ virtual void setSize(size_t sz) = 0;
+ virtual void read(void *buf, size_t count, off_t offset,
+ const std::string &D) = 0;
+ virtual void write(const void *buf, size_t count, off_t offset,
+ const std::string &D) = 0;
+
+protected:
+ std::string FileName;
+};
+}
+
+#endif //__GENERIC_FILE_IO
diff --git a/GenericIO.cxx b/GenericIO.cxx
index de66c46af8d7c0e3f20387a2ffd6bcb6bb6f05aa..77d8380e9edada948279270a3e9a995bfec1333f 100644
--- a/GenericIO.cxx
+++ b/GenericIO.cxx
@@ -1,32 +1,32 @@
/*
* Copyright (C) 2015, UChicago Argonne, LLC
* All Rights Reserved
- *
+ *
* Generic IO (ANL-15-066)
* Hal Finkel, Argonne National Laboratory
- *
+ *
* OPEN SOURCE LICENSE
- *
+ *
* Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne,
* LLC, the U.S. Government retains certain rights in this software.
- *
+ *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
- *
+ *
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
- *
+ *
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
- *
+ *
* 3. Neither the names of UChicago Argonne, LLC or the Department of Energy
* nor the names of its contributors may be used to endorse or promote
* products derived from this software without specific prior written
* permission.
- *
+ *
* *****************************************************************************
- *
+ *
* DISCLAIMER
* THE SOFTWARE IS SUPPLIED “AS IS†WITHOUT WARRANTY OF ANY KIND. NEITHER THE
* UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR
@@ -35,7 +35,7 @@
* ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS,
* PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE
* PRIVATELY OWNED RIGHTS.
- *
+ *
* *****************************************************************************
*/
@@ -407,6 +407,11 @@ void GenericIO::setFH(
else if (FileIOType == FileIOMPICollective)
FH.get() = new GenericFileIO_MPICollective(R);
else
+#endif
+#ifdef GENERICIO_WITH_VELOC
+ if (FileIOType == FileIOVELOC)
+ FH.get() = new GenericFileIO_VELOC();
+ else
#endif
FH.get() = new GenericFileIO_POSIX();
}
@@ -1861,8 +1866,7 @@ void GenericIO::getVariableInfo(vector<VariableInfo> &VI) {
void GenericIO::setNaturalDefaultPartition() {
#ifdef __bgq__
DefaultPartition = MPIX_IO_link_id();
-#else
-#ifndef GENERICIO_NO_MPI
+#elif !defined(GENERICIO_NO_MPI)
bool UseName = true;
const char *EnvStr = getenv("GENERICIO_PARTITIONS_USE_NAME");
if (EnvStr) {
@@ -1895,6 +1899,10 @@ void GenericIO::setNaturalDefaultPartition() {
}
}
#endif
+#ifdef GENERICIO_WITH_VELOC
+ const char *EnvVELOC = getenv("GENERICIO_USE_VELOC");
+ if (EnvVELOC)
+ setDefaultFileIOType(GenericIO::FileIOVELOC);
#endif
}
diff --git a/GenericIO.h b/GenericIO.h
index 37cbe77d4e31b027966a1e40753446def1323178..02bf68563c8d257d17ff35cc9e95ea49c46955d5 100644
--- a/GenericIO.h
+++ b/GenericIO.h
@@ -1,32 +1,32 @@
/*
* Copyright (C) 2015, UChicago Argonne, LLC
* All Rights Reserved
- *
+ *
* Generic IO (ANL-15-066)
* Hal Finkel, Argonne National Laboratory
- *
+ *
* OPEN SOURCE LICENSE
- *
+ *
* Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne,
* LLC, the U.S. Government retains certain rights in this software.
- *
+ *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
- *
+ *
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
- *
+ *
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
- *
+ *
* 3. Neither the names of UChicago Argonne, LLC or the Department of Energy
* nor the names of its contributors may be used to endorse or promote
* products derived from this software without specific prior written
* permission.
- *
+ *
* *****************************************************************************
- *
+ *
* DISCLAIMER
* THE SOFTWARE IS SUPPLIED “AS IS†WITHOUT WARRANTY OF ANY KIND. NEITHER THE
* UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR
@@ -35,7 +35,7 @@
* ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS,
* PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE
* PRIVATELY OWNED RIGHTS.
- *
+ *
* *****************************************************************************
*/
@@ -57,24 +57,12 @@
#include <unistd.h>
-namespace gio {
-
-class GenericFileIO {
-public:
- virtual ~GenericFileIO() {}
-
-public:
- virtual void open(const std::string &FN, bool ForReading = false, bool MustExist = false) = 0;
- virtual void setSize(size_t sz) = 0;
- virtual void read(void *buf, size_t count, off_t offset,
- const std::string &D) = 0;
- virtual void write(const void *buf, size_t count, off_t offset,
- const std::string &D) = 0;
-
-protected:
- std::string FileName;
-};
+#include "GenericFileIO.h"
+#ifdef GENERICIO_WITH_VELOC
+#include "FileIOVELOC.hpp"
+#endif
+namespace gio {
#ifndef GENERICIO_NO_MPI
class GenericFileIO_MPI : public GenericFileIO {
public:
@@ -317,6 +305,9 @@ public:
FileIOMPI,
FileIOPOSIX,
FileIOMPICollective
+#ifdef GENERICIO_WITH_VELOC
+ , FileIOVELOC
+#endif
};
#ifndef GENERICIO_NO_MPI
@@ -354,7 +345,7 @@ public:
void setNumElems(std::size_t E) {
NElems = E;
-#ifndef GENERICIO_NO_MPI
+#if !defined(GENERICIO_NO_MPI) && !defined(GENERICIO_WITH_VELOC)
int IsLarge = E >= CollectiveMPIIOThreshold;
int AllIsLarge;
MPI_Allreduce(&IsLarge, &AllIsLarge, 1, MPI_INT, MPI_SUM, Comm);
@@ -471,6 +462,12 @@ public:
static void setNaturalDefaultPartition();
+ static void flushAll() {
+#ifdef GENERICIO_WITH_VELOC
+ GenericFileIO_VELOC::flush();
+#endif
+ }
+
static void setDefaultShouldCompress(bool C) {
DefaultShouldCompress = C;
}
@@ -649,4 +646,3 @@ public:
} /* END namespace cosmotk */
#endif // GENERICIO_H
-
diff --git a/GenericIOBenchmarkWrite.cxx b/GenericIOBenchmarkWrite.cxx
index 39ca35bef5211fda351f086d9f803cb45d9d7012..749171e5110a420830d47c4ae7a8f481e1074faf 100644
--- a/GenericIOBenchmarkWrite.cxx
+++ b/GenericIOBenchmarkWrite.cxx
@@ -1,32 +1,32 @@
/*
* Copyright (C) 2015, UChicago Argonne, LLC
* All Rights Reserved
- *
+ *
* Generic IO (ANL-15-066)
* Hal Finkel, Argonne National Laboratory
- *
+ *
* OPEN SOURCE LICENSE
- *
+ *
* Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne,
* LLC, the U.S. Government retains certain rights in this software.
- *
+ *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
- *
+ *
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
- *
+ *
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
- *
+ *
* 3. Neither the names of UChicago Argonne, LLC or the Department of Energy
* nor the names of its contributors may be used to endorse or promote
* products derived from this software without specific prior written
* permission.
- *
+ *
* *****************************************************************************
- *
+ *
* DISCLAIMER
* THE SOFTWARE IS SUPPLIED “AS IS†WITHOUT WARRANTY OF ANY KIND. NEITHER THE
* UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR
@@ -35,7 +35,7 @@
* ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS,
* PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE
* PRIVATELY OWNED RIGHTS.
- *
+ *
* *****************************************************************************
*/
@@ -148,6 +148,11 @@ int main(int argc, char *argv[]) {
const char *EnvStr = getenv("GENERICIO_USE_MPIIO");
if (EnvStr && string(EnvStr) == "1")
Method = GenericIO::FileIOMPI;
+#ifdef GENERICIO_WITH_VELOC
+ EnvStr = getenv("GENERICIO_USE_VELOC");
+ if (EnvStr)
+ Method = GenericIO::FileIOVELOC;
+#endif
{ // scope GIO
GenericIO GIO(
@@ -215,6 +220,7 @@ int main(int argc, char *argv[]) {
GIO.write();
} // destroy GIO prior to calling MPI_Finalize
+ GenericIO::flushAll();
if (UseAOS) {
pos.resize(Np);
@@ -236,4 +242,3 @@ int main(int argc, char *argv[]) {
return 0;
}
-
diff --git a/README.md b/README.md
index 659ea68f9935f15feea650caeaff6d7479016f4f..f585b058ab6038c224368ac68d037ed16d12b097 100644
--- a/README.md
+++ b/README.md
@@ -102,8 +102,23 @@ compiler path as
CC=/path/to/gcc CXX=/path/to/g++ python -m pip install .
```
-
If the compiler supports OpenMP, the library will be threaded. Make
sure to set ``OMP_NUM_THREADS`` to an appropriate variable, in particluar when
using multiple MPI ranks per node.
+## Installing and running with VELOC support
+
+**Requirements**
+
+This mode requires a working VELOC installation.
+Instructions can be found here: [https://veloc.readthedocs.io](https://veloc.readthedocs.io)
+
+**Install**
+
+Set the VELOC_INSTALL_DIR variable in GNUMakefile to the root of the VELOC installation directory.
+Then proceed to compile and link GIO as usual.
+
+**Run**
+
+Define the GENERICIO_USE_VELOC environment variable as the path to the VELOC configuration file.
+An example is available here: veloc.cfg
diff --git a/thirdparty/veloc/FileIOVELOC.cxx b/thirdparty/veloc/FileIOVELOC.cxx
new file mode 100644
index 0000000000000000000000000000000000000000..e0114c3a620044c812927f1582496bd5e904f0a5
--- /dev/null
+++ b/thirdparty/veloc/FileIOVELOC.cxx
@@ -0,0 +1,48 @@
+#include "FileIOVELOC.hpp"
+
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#define __DEBUG
+#include "debug.hpp"
+
+static std::string get_veloc_cfg() {
+ char *ptr = getenv("GENERICIO_USE_VELOC");
+ if (ptr == NULL)
+ return "";
+ return ptr;
+}
+
+namespace gio {
+
+GenericFileIO_VELOC::GenericFileIO_VELOC() : cached_file(get_veloc_cfg()) {
+}
+
+GenericFileIO_VELOC::~GenericFileIO_VELOC() { }
+
+void GenericFileIO_VELOC::open(const std::string &FN, bool ForReading, bool MustExist) {
+ int flags = ForReading ? O_RDONLY : (O_WRONLY | (!MustExist ? O_CREAT : 0));
+ int mode = S_IRUSR | S_IWUSR | S_IRGRP;
+ FileName = FN;
+ cached_file.open(FN, flags, mode);
+}
+
+void GenericFileIO_VELOC::setSize(size_t sz) { }
+
+void GenericFileIO_VELOC::read(void *buf, size_t count, off_t offset, const std::string &D) {
+ if (!cached_file.pread(buf, count, offset))
+ throw std::runtime_error("Unable to read " + D + " from file: " + FileName + ": " + strerror(errno));
+}
+
+void GenericFileIO_VELOC::write(const void *buf, size_t count, off_t offset, const std::string &D) {
+ if (!cached_file.pwrite(buf, count, offset))
+ throw std::runtime_error("Unable to write " + D + " to file: " + FileName + ": " + strerror(errno));
+}
+
+void GenericFileIO_VELOC::flush() {
+ veloc::cached_file_t::flush();
+ veloc::cached_file_t::shutdown();
+}
+
+}
diff --git a/thirdparty/veloc/FileIOVELOC.hpp b/thirdparty/veloc/FileIOVELOC.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..66b791ded1f6d4cfa9a68d0092fafebd4b1e543c
--- /dev/null
+++ b/thirdparty/veloc/FileIOVELOC.hpp
@@ -0,0 +1,25 @@
+#ifndef __FILE_IO_VELOC
+#define __FILE_IO_VELOC
+
+#include "GenericFileIO.h"
+#include "veloc/cache.hpp"
+
+namespace gio {
+class GenericFileIO_VELOC : public GenericFileIO {
+ veloc::cached_file_t cached_file;
+
+public:
+ GenericFileIO_VELOC();
+ ~GenericFileIO_VELOC();
+ void open(const std::string &FN, bool ForReading = false, bool MustExist = false);
+ void setSize(size_t sz);
+ void read(void *buf, size_t count, off_t offset, const std::string &D);
+ void write(const void *buf, size_t count, off_t offset, const std::string &D);
+ static void flush();
+
+protected:
+ int FH;
+};
+}
+
+#endif //__FILE_IO_VELOC
diff --git a/thirdparty/veloc/debug.hpp b/thirdparty/veloc/debug.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..e29be3d36613385d3e0e660fed5fd4c9c30356e7
--- /dev/null
+++ b/thirdparty/veloc/debug.hpp
@@ -0,0 +1,73 @@
+#ifndef __DEBUG_CONFIG
+#define __DEBUG_CONFIG
+
+#include <iostream>
+#include <chrono>
+#include <mutex>
+#include <cstring>
+
+extern std::ostream *logger;
+static auto beginning = std::chrono::steady_clock::now();
+static std::mutex log_mutex;
+
+#ifdef __BENCHMARK
+#define TIMER_START(timer) auto timer = std::chrono::steady_clock::now();
+#define TIMER_STOP(timer, message) {\
+ auto now = std::chrono::steady_clock::now();\
+ auto d = std::chrono::duration_cast<std::chrono::microseconds>(now - timer).count();\
+ auto t = std::chrono::duration_cast<std::chrono::seconds>(now - beginning).count();\
+ std::unique_lock<std::mutex> lock(log_mutex);\
+ (*logger) << "[BENCHMARK " << t << "] [" << __FILE__ << ":" << __LINE__ << ":" << __FUNCTION__ << "] [time elapsed: " << d << " us] " << message << std::endl;\
+ }
+#else
+#define TIMER_START(timer)
+#define TIMER_STOP(timer, message)
+#endif
+
+#define MESSAGE(level, message) {\
+ std::unique_lock<std::mutex> lock(log_mutex);\
+ (*logger) << "[" << level << " " << std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - beginning).count() << "] [" \
+ << __FILE__ << ":" << __LINE__ << ":" << __FUNCTION__ << "] " << message << std::endl;\
+}
+
+#define FATAL(message) {\
+ MESSAGE("FATAL", message);\
+ exit(-1);\
+}
+
+#ifdef __INFO
+#define __ERROR
+#define INFO(message) MESSAGE("INFO", message)
+#else
+#define INFO(message)
+#endif
+
+#ifdef __ERROR
+#define ERROR(message) MESSAGE("ERROR", message)
+#else
+#define ERROR(message)
+#endif
+
+#ifdef __ASSERT
+#define ASSERT(expression) {\
+ if (!(expression)) {\
+ MESSAGE("ASSERT", "failed on expression: " << #expression);\
+ exit(-2);\
+ }\
+ }
+#else
+#define ASSERT(expression)
+#endif
+
+#endif
+
+#undef DBG
+#undef DBG_COND
+#ifdef __DEBUG
+#define DBG(message) MESSAGE("DEBUG", message)
+#define DBG_COND(cond, message) if (cond) DBG(message)
+#undef __DEBUG
+#else
+#define DBG(message)
+#define DBG_COND(cond, message)
+#endif
diff --git a/veloc.cfg b/veloc.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..cadd6f268248b250a29db8325b4a1fe8099a1640
--- /dev/null
+++ b/veloc.cfg
@@ -0,0 +1,3 @@
+scratch = /tmp/scratch
+persistent = /tmp/persistent
+mode = async
\ No newline at end of file