From ee74841c23f29fd051de5e8f8358843887b241e2 Mon Sep 17 00:00:00 2001
From: Bogdan Nicolae <bogdan.nicolae@acm.org>
Date: Fri, 7 Oct 2022 23:09:54 -0500
Subject: [PATCH] added initial VELOC support
---
GNUmakefile | 22 ++++++++--
GenericFileIO.h | 24 +++++++++++
GenericIO.cxx | 34 +++++++++------
GenericIO.h | 54 +++++++++++------------
GenericIOBenchmarkWrite.cxx | 27 +++++++-----
README.md | 17 +++++++-
thirdparty/veloc/FileIOVELOC.cxx | 49 +++++++++++++++++++++
thirdparty/veloc/FileIOVELOC.hpp | 25 +++++++++++
thirdparty/veloc/debug.hpp | 73 ++++++++++++++++++++++++++++++++
veloc.cfg | 3 ++
10 files changed, 272 insertions(+), 56 deletions(-)
create mode 100644 GenericFileIO.h
create mode 100644 thirdparty/veloc/FileIOVELOC.cxx
create mode 100644 thirdparty/veloc/FileIOVELOC.hpp
create mode 100644 thirdparty/veloc/debug.hpp
create mode 100644 veloc.cfg
diff --git a/GNUmakefile b/GNUmakefile
index 4881c6e..16725c8 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -46,6 +46,8 @@ MPICXX = mpicxx
all: fe-progs mpi-progs
sql: fe-sqlite
+VELOC_INSTALL_DIR := $(HOME)/deploy
+
BLOSC_CPPFLAGS := \
-Ithirdparty/blosc \
-DHAVE_LZ4 -DHAVE_SNAPPY -DHAVE_ZLIB -DHAVE_ZSTD \
@@ -60,6 +62,16 @@ BLOSC_CPPFLAGS := \
-Ithirdparty/blosc/internal-complibs/zstd-0.7.4/decompress \
-Ithirdparty/SZ/sz/include
+ifdef VELOC_INSTALL_DIR
+BLOSC_CPPFLAGS := $(BLOSC_CPPFLAGS) \
+ -DGENERICIO_WITH_VELOC \
+ -Ithirdparty/veloc \
+ -I$(VELOC_INSTALL_DIR)/include
+BASE_CXXFLAGS := \
+ -L$(VELOC_INSTALL_DIR)/lib -L$(VELOC_INSTALL_DIR)/lib64 \
+ -Wl,--copy-dt-needed-entries -lveloc-client
+endif
+
BASE_CPPFLAGS := $(BLOSC_CPPFLAGS) -I. -D__STDC_CONSTANT_MACROS
UNAME_S := $(shell uname -s)
@@ -75,12 +87,12 @@ endif
FEDIR = frontend
FE_CFLAGS := -g -fPIC -O3 $(OPENMPFLAG) $(G99FLAG)
-FE_CXXFLAGS := -g -fPIC -O3 $(OPENMPFLAG)
+FE_CXXFLAGS := -g -fPIC -O3 $(OPENMPFLAG) $(BASE_CXXFLAGS)
FE_CPPFLAGS := $(BASE_CPPFLAGS) -Ithirdparty/sqlite -DGENERICIO_NO_MPI
MPIDIR = mpi
MPI_CFLAGS := -g -O3 $(OPENMPFLAG) $(G99FLAG)
-MPI_CXXFLAGS := -g -O3 $(OPENMPFLAG)
+MPI_CXXFLAGS := -g -O3 $(OPENMPFLAG) $(BASE_CXXFLAGS)
MPI_CPPFLAGS := $(BASE_CPPFLAGS)
$(FEDIR):
@@ -189,6 +201,11 @@ BLOSC_O := \
thirdparty/SZ/sz/src/szd_double_ts.o \
thirdparty/SZ/sz/src/szd_float_ts.o
+ifdef VELOC_INSTALL_DIR
+BLOSC_O := $(BLOSC_O) \
+ thirdparty/veloc/FileIOVELOC.o
+endif
+
FE_BLOSC_O := $(addprefix $(FEDIR)/,$(BLOSC_O))
$(FEDIR)/GenericIOPrint: $(FEDIR)/GenericIOPrint.o $(FEDIR)/GenericIO.o $(FE_BLOSC_O)
@@ -259,4 +276,3 @@ fe-sqlite: frontend-sqlite
clean:
rm -rf frontend mpi legacy_python/genericio.pyc
-
diff --git a/GenericFileIO.h b/GenericFileIO.h
new file mode 100644
index 0000000..33d6fe3
--- /dev/null
+++ b/GenericFileIO.h
@@ -0,0 +1,24 @@
+#ifndef __GENERIC_FILE_IO
+#define __GENERIC_FILE_IO
+
+#include <string>
+
+namespace gio {
+class GenericFileIO {
+public:
+ virtual ~GenericFileIO() {}
+
+public:
+ virtual void open(const std::string &FN, bool ForReading = false, bool MustExist = false) = 0;
+ virtual void setSize(size_t sz) = 0;
+ virtual void read(void *buf, size_t count, off_t offset,
+ const std::string &D) = 0;
+ virtual void write(const void *buf, size_t count, off_t offset,
+ const std::string &D) = 0;
+
+protected:
+ std::string FileName;
+};
+}
+
+#endif //__GENERIC_FILE_IO
diff --git a/GenericIO.cxx b/GenericIO.cxx
index de66c46..e386cdf 100644
--- a/GenericIO.cxx
+++ b/GenericIO.cxx
@@ -1,32 +1,32 @@
/*
* Copyright (C) 2015, UChicago Argonne, LLC
* All Rights Reserved
- *
+ *
* Generic IO (ANL-15-066)
* Hal Finkel, Argonne National Laboratory
- *
+ *
* OPEN SOURCE LICENSE
- *
+ *
* Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne,
* LLC, the U.S. Government retains certain rights in this software.
- *
+ *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
- *
+ *
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
- *
+ *
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
- *
+ *
* 3. Neither the names of UChicago Argonne, LLC or the Department of Energy
* nor the names of its contributors may be used to endorse or promote
* products derived from this software without specific prior written
* permission.
- *
+ *
* *****************************************************************************
- *
+ *
* DISCLAIMER
* THE SOFTWARE IS SUPPLIED “AS IS†WITHOUT WARRANTY OF ANY KIND. NEITHER THE
* UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR
@@ -35,10 +35,12 @@
* ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS,
* PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE
* PRIVATELY OWNED RIGHTS.
- *
+ *
* *****************************************************************************
*/
+#include <mpi.h>
+
#define _XOPEN_SOURCE 600
#include "CRC64.h"
#include "GenericIO.h"
@@ -407,6 +409,11 @@ void GenericIO::setFH(
else if (FileIOType == FileIOMPICollective)
FH.get() = new GenericFileIO_MPICollective(R);
else
+#endif
+#ifdef GENERICIO_WITH_VELOC
+ if (FileIOType == FileIOVELOC)
+ FH.get() = new GenericFileIO_VELOC();
+ else
#endif
FH.get() = new GenericFileIO_POSIX();
}
@@ -1861,8 +1868,7 @@ void GenericIO::getVariableInfo(vector<VariableInfo> &VI) {
void GenericIO::setNaturalDefaultPartition() {
#ifdef __bgq__
DefaultPartition = MPIX_IO_link_id();
-#else
-#ifndef GENERICIO_NO_MPI
+#elif !defined(GENERICIO_NO_MPI)
bool UseName = true;
const char *EnvStr = getenv("GENERICIO_PARTITIONS_USE_NAME");
if (EnvStr) {
@@ -1895,6 +1901,10 @@ void GenericIO::setNaturalDefaultPartition() {
}
}
#endif
+#ifdef GENERICIO_WITH_VELOC
+ const char *EnvVELOC = getenv("GENERICIO_USE_VELOC");
+ if (EnvVELOC)
+ setDefaultFileIOType(GenericIO::FileIOVELOC);
#endif
}
diff --git a/GenericIO.h b/GenericIO.h
index 37cbe77..02bf685 100644
--- a/GenericIO.h
+++ b/GenericIO.h
@@ -1,32 +1,32 @@
/*
* Copyright (C) 2015, UChicago Argonne, LLC
* All Rights Reserved
- *
+ *
* Generic IO (ANL-15-066)
* Hal Finkel, Argonne National Laboratory
- *
+ *
* OPEN SOURCE LICENSE
- *
+ *
* Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne,
* LLC, the U.S. Government retains certain rights in this software.
- *
+ *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
- *
+ *
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
- *
+ *
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
- *
+ *
* 3. Neither the names of UChicago Argonne, LLC or the Department of Energy
* nor the names of its contributors may be used to endorse or promote
* products derived from this software without specific prior written
* permission.
- *
+ *
* *****************************************************************************
- *
+ *
* DISCLAIMER
* THE SOFTWARE IS SUPPLIED “AS IS†WITHOUT WARRANTY OF ANY KIND. NEITHER THE
* UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR
@@ -35,7 +35,7 @@
* ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS,
* PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE
* PRIVATELY OWNED RIGHTS.
- *
+ *
* *****************************************************************************
*/
@@ -57,24 +57,12 @@
#include <unistd.h>
-namespace gio {
-
-class GenericFileIO {
-public:
- virtual ~GenericFileIO() {}
-
-public:
- virtual void open(const std::string &FN, bool ForReading = false, bool MustExist = false) = 0;
- virtual void setSize(size_t sz) = 0;
- virtual void read(void *buf, size_t count, off_t offset,
- const std::string &D) = 0;
- virtual void write(const void *buf, size_t count, off_t offset,
- const std::string &D) = 0;
-
-protected:
- std::string FileName;
-};
+#include "GenericFileIO.h"
+#ifdef GENERICIO_WITH_VELOC
+#include "FileIOVELOC.hpp"
+#endif
+namespace gio {
#ifndef GENERICIO_NO_MPI
class GenericFileIO_MPI : public GenericFileIO {
public:
@@ -317,6 +305,9 @@ public:
FileIOMPI,
FileIOPOSIX,
FileIOMPICollective
+#ifdef GENERICIO_WITH_VELOC
+ , FileIOVELOC
+#endif
};
#ifndef GENERICIO_NO_MPI
@@ -354,7 +345,7 @@ public:
void setNumElems(std::size_t E) {
NElems = E;
-#ifndef GENERICIO_NO_MPI
+#if !defined(GENERICIO_NO_MPI) && !defined(GENERICIO_WITH_VELOC)
int IsLarge = E >= CollectiveMPIIOThreshold;
int AllIsLarge;
MPI_Allreduce(&IsLarge, &AllIsLarge, 1, MPI_INT, MPI_SUM, Comm);
@@ -471,6 +462,12 @@ public:
static void setNaturalDefaultPartition();
+ static void flushAll() {
+#ifdef GENERICIO_WITH_VELOC
+ GenericFileIO_VELOC::flush();
+#endif
+ }
+
static void setDefaultShouldCompress(bool C) {
DefaultShouldCompress = C;
}
@@ -649,4 +646,3 @@ public:
} /* END namespace cosmotk */
#endif // GENERICIO_H
-
diff --git a/GenericIOBenchmarkWrite.cxx b/GenericIOBenchmarkWrite.cxx
index 39ca35b..749171e 100644
--- a/GenericIOBenchmarkWrite.cxx
+++ b/GenericIOBenchmarkWrite.cxx
@@ -1,32 +1,32 @@
/*
* Copyright (C) 2015, UChicago Argonne, LLC
* All Rights Reserved
- *
+ *
* Generic IO (ANL-15-066)
* Hal Finkel, Argonne National Laboratory
- *
+ *
* OPEN SOURCE LICENSE
- *
+ *
* Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne,
* LLC, the U.S. Government retains certain rights in this software.
- *
+ *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
- *
+ *
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
- *
+ *
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
- *
+ *
* 3. Neither the names of UChicago Argonne, LLC or the Department of Energy
* nor the names of its contributors may be used to endorse or promote
* products derived from this software without specific prior written
* permission.
- *
+ *
* *****************************************************************************
- *
+ *
* DISCLAIMER
* THE SOFTWARE IS SUPPLIED “AS IS†WITHOUT WARRANTY OF ANY KIND. NEITHER THE
* UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR
@@ -35,7 +35,7 @@
* ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS,
* PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE
* PRIVATELY OWNED RIGHTS.
- *
+ *
* *****************************************************************************
*/
@@ -148,6 +148,11 @@ int main(int argc, char *argv[]) {
const char *EnvStr = getenv("GENERICIO_USE_MPIIO");
if (EnvStr && string(EnvStr) == "1")
Method = GenericIO::FileIOMPI;
+#ifdef GENERICIO_WITH_VELOC
+ EnvStr = getenv("GENERICIO_USE_VELOC");
+ if (EnvStr)
+ Method = GenericIO::FileIOVELOC;
+#endif
{ // scope GIO
GenericIO GIO(
@@ -215,6 +220,7 @@ int main(int argc, char *argv[]) {
GIO.write();
} // destroy GIO prior to calling MPI_Finalize
+ GenericIO::flushAll();
if (UseAOS) {
pos.resize(Np);
@@ -236,4 +242,3 @@ int main(int argc, char *argv[]) {
return 0;
}
-
diff --git a/README.md b/README.md
index 659ea68..f585b05 100644
--- a/README.md
+++ b/README.md
@@ -102,8 +102,23 @@ compiler path as
CC=/path/to/gcc CXX=/path/to/g++ python -m pip install .
```
-
If the compiler supports OpenMP, the library will be threaded. Make
sure to set ``OMP_NUM_THREADS`` to an appropriate variable, in particluar when
using multiple MPI ranks per node.
+## Installing and running with VELOC support
+
+**Requirements**
+
+This mode requires a working VELOC installation.
+Instructions can be found here: [https://veloc.readthedocs.io](https://veloc.readthedocs.io)
+
+**Install**
+
+Set the VELOC_INSTALL_DIR variable in GNUMakefile to the root of the VELOC installation directory.
+Then proceed to compile and link GIO as usual.
+
+**Run**
+
+Define the GENERICIO_USE_VELOC environment variable as the path to the VELOC configuration file.
+An example is available here: veloc.cfg
diff --git a/thirdparty/veloc/FileIOVELOC.cxx b/thirdparty/veloc/FileIOVELOC.cxx
new file mode 100644
index 0000000..85e4d49
--- /dev/null
+++ b/thirdparty/veloc/FileIOVELOC.cxx
@@ -0,0 +1,49 @@
+#include "FileIOVELOC.hpp"
+
+#include <stdlib.h>
+#include <mpi.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#define __DEBUG
+#include "debug.hpp"
+
+static std::string get_veloc_cfg() {
+ char *ptr = getenv("GENERICIO_USE_VELOC");
+ if (ptr == NULL)
+ FATAL("GENERICIO_USE_VELOC environment variable missing, must point to a valid VELOC configuration file");
+ return std::string(ptr);
+}
+
+namespace gio {
+
+GenericFileIO_VELOC::GenericFileIO_VELOC() : cached_file(get_veloc_cfg()) {
+}
+
+GenericFileIO_VELOC::~GenericFileIO_VELOC() { }
+
+void GenericFileIO_VELOC::open(const std::string &FN, bool ForReading, bool MustExist) {
+ int flags = ForReading ? O_RDONLY : (O_WRONLY | (!MustExist ? O_CREAT : 0));
+ int mode = S_IRUSR | S_IWUSR | S_IRGRP;
+ FileName = FN;
+ cached_file.open(FN, flags, mode);
+}
+
+void GenericFileIO_VELOC::setSize(size_t sz) { }
+
+void GenericFileIO_VELOC::read(void *buf, size_t count, off_t offset, const std::string &D) {
+ if (!cached_file.pread(buf, count, offset))
+ throw std::runtime_error("Unable to read " + D + " from file: " + FileName + ": " + strerror(errno));
+}
+
+void GenericFileIO_VELOC::write(const void *buf, size_t count, off_t offset, const std::string &D) {
+ if (!cached_file.pwrite(buf, count, offset))
+ throw std::runtime_error("Unable to write " + D + " to file: " + FileName + ": " + strerror(errno));
+}
+
+void GenericFileIO_VELOC::flush() {
+ veloc::cached_file_t::flush();
+ veloc::cached_file_t::shutdown();
+}
+
+}
diff --git a/thirdparty/veloc/FileIOVELOC.hpp b/thirdparty/veloc/FileIOVELOC.hpp
new file mode 100644
index 0000000..66b791d
--- /dev/null
+++ b/thirdparty/veloc/FileIOVELOC.hpp
@@ -0,0 +1,25 @@
+#ifndef __FILE_IO_VELOC
+#define __FILE_IO_VELOC
+
+#include "GenericFileIO.h"
+#include "veloc/cache.hpp"
+
+namespace gio {
+class GenericFileIO_VELOC : public GenericFileIO {
+ veloc::cached_file_t cached_file;
+
+public:
+ GenericFileIO_VELOC();
+ ~GenericFileIO_VELOC();
+ void open(const std::string &FN, bool ForReading = false, bool MustExist = false);
+ void setSize(size_t sz);
+ void read(void *buf, size_t count, off_t offset, const std::string &D);
+ void write(const void *buf, size_t count, off_t offset, const std::string &D);
+ static void flush();
+
+protected:
+ int FH;
+};
+}
+
+#endif //__FILE_IO_VELOC
diff --git a/thirdparty/veloc/debug.hpp b/thirdparty/veloc/debug.hpp
new file mode 100644
index 0000000..e29be3d
--- /dev/null
+++ b/thirdparty/veloc/debug.hpp
@@ -0,0 +1,73 @@
+#ifndef __DEBUG_CONFIG
+#define __DEBUG_CONFIG
+
+#include <iostream>
+#include <chrono>
+#include <mutex>
+#include <cstring>
+
+extern std::ostream *logger;
+static auto beginning = std::chrono::steady_clock::now();
+static std::mutex log_mutex;
+
+#ifdef __BENCHMARK
+#define TIMER_START(timer) auto timer = std::chrono::steady_clock::now();
+#define TIMER_STOP(timer, message) {\
+ auto now = std::chrono::steady_clock::now();\
+ auto d = std::chrono::duration_cast<std::chrono::microseconds>(now - timer).count();\
+ auto t = std::chrono::duration_cast<std::chrono::seconds>(now - beginning).count();\
+ std::unique_lock<std::mutex> lock(log_mutex);\
+ (*logger) << "[BENCHMARK " << t << "] [" << __FILE__ << ":" << __LINE__ << ":" << __FUNCTION__ << "] [time elapsed: " << d << " us] " << message << std::endl;\
+ }
+#else
+#define TIMER_START(timer)
+#define TIMER_STOP(timer, message)
+#endif
+
+#define MESSAGE(level, message) {\
+ std::unique_lock<std::mutex> lock(log_mutex);\
+ (*logger) << "[" << level << " " << std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - beginning).count() << "] [" \
+ << __FILE__ << ":" << __LINE__ << ":" << __FUNCTION__ << "] " << message << std::endl;\
+}
+
+#define FATAL(message) {\
+ MESSAGE("FATAL", message);\
+ exit(-1);\
+}
+
+#ifdef __INFO
+#define __ERROR
+#define INFO(message) MESSAGE("INFO", message)
+#else
+#define INFO(message)
+#endif
+
+#ifdef __ERROR
+#define ERROR(message) MESSAGE("ERROR", message)
+#else
+#define ERROR(message)
+#endif
+
+#ifdef __ASSERT
+#define ASSERT(expression) {\
+ if (!(expression)) {\
+ MESSAGE("ASSERT", "failed on expression: " << #expression);\
+ exit(-2);\
+ }\
+ }
+#else
+#define ASSERT(expression)
+#endif
+
+#endif
+
+#undef DBG
+#undef DBG_COND
+#ifdef __DEBUG
+#define DBG(message) MESSAGE("DEBUG", message)
+#define DBG_COND(cond, message) if (cond) DBG(message)
+#undef __DEBUG
+#else
+#define DBG(message)
+#define DBG_COND(cond, message)
+#endif
diff --git a/veloc.cfg b/veloc.cfg
new file mode 100644
index 0000000..cadd6f2
--- /dev/null
+++ b/veloc.cfg
@@ -0,0 +1,3 @@
+scratch = /tmp/scratch
+persistent = /tmp/persistent
+mode = async
\ No newline at end of file
--
GitLab