diff --git a/new_python/CMakeLists.txt b/new_python/CMakeLists.txt
index f9b226d2b97e682a0de7762e2d664c93c200e9ef..78e047c3a4a52e35ed7a0240a6d7012f1690ef10 100644
--- a/new_python/CMakeLists.txt
+++ b/new_python/CMakeLists.txt
@@ -14,10 +14,14 @@ if(NOT pybind11_POPULATED)
add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR})
endif()
+# the mpi version
+if(MPI_FOUND)
pybind11_add_module(pygio_new genericio.cpp)
set_target_properties(pygio_new PROPERTIES OUTPUT_NAME pygio)
-if(MPI_FOUND)
- target_link_libraries(pygio_new PRIVATE genericio_mpi)
-else()
- target_link_libraries(pygio_new PRIVATE genericio)
-endif()
\ No newline at end of file
+target_link_libraries(pygio_new PRIVATE genericio_mpi)
+endif()
+
+# The no-mpi version
+pybind11_add_module(pygio_new_nompi genericio.cpp)
+set_target_properties(pygio_new_nompi PROPERTIES OUTPUT_NAME pygio_nompi)
+target_link_libraries(pygio_new_nompi PRIVATE genericio)
\ No newline at end of file
diff --git a/new_python/README.md b/new_python/README.md
index 6c88db9eeb55aa7f03589b421bb326dc3677a9fc..b14a2ced5af8afab9f93e30cb85a18e8ce19bd1d 100644
--- a/new_python/README.md
+++ b/new_python/README.md
@@ -136,4 +136,20 @@ rank 4 read 70625 elements
rank 5 read 70625 elements
rank 6 read 70625 elements
rank 7 read 70625 elements
-```
\ No newline at end of file
+```
+
+### Force the non-MPI version to be imported
+
+Some clusters prevent the loading of MPI libraries on the login nodes. In order
+to still be able to use the same `pygio` on the login nodes, the non-MPI library
+can be loaded by setting the `GENERICIO_NO_MPI` environment variable before
+importing `pygio`:
+
+```python
+import os
+os.environ['GENERICIO_NO_MPI'] = 'True'
+import pygio
+```
+
+A warning will be printed that the writing capabilities of genericio are not
+available in this mode.
\ No newline at end of file
diff --git a/new_python/genericio.cpp b/new_python/genericio.cpp
index e8ad1d960a7a85a9b13e36623f35c8d8315dc23a..59cf08b5e5b50c2019634cd07ba18cfaf3b6df1d 100644
--- a/new_python/genericio.cpp
+++ b/new_python/genericio.cpp
@@ -136,20 +136,37 @@ private:
std::vector<gio::GenericIO::VariableInfo> variables;
};
-std::map<std::string, py::array> read_genericio(std::string filename, std::optional<std::vector<std::string>> var_names, PyGenericIO::FileIO method=PyGenericIO::FileIO::FileIOPOSIX, PyGenericIO::MismatchBehavior redistribute=PyGenericIO::MismatchBehavior::MismatchRedistribute, bool rebalance_source_ranks=false) {
+std::map<std::string, py::array> read_genericio(
+ std::string filename,
+ std::optional<std::vector<std::string>> var_names,
+ PyGenericIO::FileIO method=PyGenericIO::FileIO::FileIOPOSIX,
+ PyGenericIO::MismatchBehavior redistribute=PyGenericIO::MismatchBehavior::MismatchRedistribute,
+ bool rebalance_source_ranks=false
+ ) {
PyGenericIO reader(filename, method, redistribute);
+#ifndef GENERICIO_NO_MPI
if(rebalance_source_ranks)
reader.rebalanceSourceRanks();
+#endif
return reader.read(var_names);
}
-void inspect_genericio(std::string filename, PyGenericIO::FileIO method=PyGenericIO::FileIO::FileIOPOSIX, PyGenericIO::MismatchBehavior redistribute=PyGenericIO::MismatchBehavior::MismatchRedistribute) {
+void inspect_genericio(
+ std::string filename,
+ PyGenericIO::FileIO method=PyGenericIO::FileIO::FileIOPOSIX,
+ PyGenericIO::MismatchBehavior redistribute=PyGenericIO::MismatchBehavior::MismatchRedistribute
+ ) {
PyGenericIO reader(filename, method, redistribute);
reader.inspect();
}
#ifndef GENERICIO_NO_MPI
-void write_genericio(std::string filename, std::map<std::string, py::array> variables, std::array<double, 3> phys_scale, std::array<double, 3> phys_origin, PyGenericIO::FileIO method=PyGenericIO::FileIO::FileIOPOSIX) {
+void write_genericio(
+ std::string filename,
+ std::map<std::string, py::array> variables,
+ std::array<double, 3> phys_scale, std::array<double, 3> phys_origin,
+ PyGenericIO::FileIO method=PyGenericIO::FileIO::FileIOPOSIX
+ ) {
// check data integrity, find particle count
int64_t particle_count = -1;
for(auto const& [name, data]: variables) {
@@ -193,9 +210,13 @@ void write_genericio(std::string filename, std::map<std::string, py::array> vari
}
#endif
+
+#ifdef GENERICIO_NO_MPI
+PYBIND11_MODULE(pygio_nompi, m) {
+ m.doc() = "genericio python module (no MPI support)";
+#else // GENERICIO_NO_MPI
PYBIND11_MODULE(pygio, m) {
- m.doc() = "genericio python module";
-#ifndef GENERICIO_NO_MPI
+ m.doc() = "genericio python module (with MPI support)";
m.def("_init_mpi", [](){
int initialized;
MPI_Initialized(&initialized);
@@ -218,7 +239,11 @@ PYBIND11_MODULE(pygio, m) {
.value("MismatchDisallowed", PyGenericIO::MismatchBehavior::MismatchDisallowed)
.value("MismatchRedistribute", PyGenericIO::MismatchBehavior::MismatchRedistribute);
- pyGenericIO.def(py::init<std::string, PyGenericIO::FileIO, PyGenericIO::MismatchBehavior>(), py::arg("filename"), py::arg("method")=PyGenericIO::FileIO::FileIOPOSIX, py::arg("redistribute")=PyGenericIO::MismatchBehavior::MismatchRedistribute)
+ pyGenericIO.def(
+ py::init<std::string, PyGenericIO::FileIO, PyGenericIO::MismatchBehavior>(),
+ py::arg("filename"),
+ py::arg("method")=PyGenericIO::FileIO::FileIOPOSIX,
+ py::arg("redistribute")=PyGenericIO::MismatchBehavior::MismatchRedistribute)
.def("inspect", &PyGenericIO::inspect, "Print variable infos and size of GenericIO file")
.def("get_variables", &PyGenericIO::get_variables, "Get a list of VariableInformations defined in the GenericIO file")
.def("read_num_elems", (size_t (PyGenericIO::*)(int))(&PyGenericIO::readNumElems), py::arg("eff_rank")=-1)
@@ -227,7 +252,10 @@ PYBIND11_MODULE(pygio, m) {
.def("read_phys_scale", &PyGenericIO::read_phys_scale)
.def("read", &PyGenericIO::read, py::arg("variables")=nullptr)
.def("get_source_ranks", &PyGenericIO::getSourceRanks)
- .def("rebalance_source_ranks", &PyGenericIO::rebalanceSourceRanks);
+#ifndef GENERICIO_NO_MPI
+ .def("rebalance_source_ranks", &PyGenericIO::rebalanceSourceRanks)
+#endif
+ ;
py::class_<gio::GenericIO::VariableInfo>(pyGenericIO, "VariableInfo")
.def_readonly("name", &gio::GenericIO::VariableInfo::Name)
@@ -238,9 +266,29 @@ PYBIND11_MODULE(pygio, m) {
return std::string("<PyGenericIO.VariableInfo type=") +
(vi.IsFloat ? "float" : "int") + " name='" + vi.Name + "'>";
});
- m.def("read_genericio", &read_genericio, py::arg("filename"), py::arg("variables")=nullptr, py::arg("method")=PyGenericIO::FileIO::FileIOPOSIX, py::arg("redistribute")=PyGenericIO::MismatchBehavior::MismatchRedistribute, py::arg("rebalance_sourceranks")=false, py::return_value_policy::move);
- m.def("inspect_genericio", &inspect_genericio, py::arg("filename"), py::arg("method")=PyGenericIO::FileIO::FileIOPOSIX, py::arg("redistribute")=PyGenericIO::MismatchBehavior::MismatchRedistribute);
+
+ m.def("read_genericio", &read_genericio,
+ py::arg("filename"),
+ py::arg("variables")=nullptr,
+ py::kw_only(),
+ py::arg("method")=PyGenericIO::FileIO::FileIOPOSIX,
+ py::arg("redistribute")=PyGenericIO::MismatchBehavior::MismatchRedistribute,
+ py::arg("rebalance_sourceranks")=false,
+ py::return_value_policy::move);
+
+ m.def("inspect_genericio", &inspect_genericio,
+ py::arg("filename"),
+ py::kw_only(),
+ py::arg("method")=PyGenericIO::FileIO::FileIOPOSIX,
+ py::arg("redistribute")=PyGenericIO::MismatchBehavior::MismatchRedistribute);
+
#ifndef GENERICIO_NO_MPI
- m.def("write_genericio", &write_genericio, py::arg("filename"), py::arg("variables"), py::arg("phys_scale"), py::arg("phys_origin") = std::array<double, 3>({0., 0., 0.}), py::arg("method")=PyGenericIO::FileIO::FileIOPOSIX);
+ m.def("write_genericio", &write_genericio,
+ py::arg("filename"),
+ py::arg("variables"),
+ py::arg("phys_scale"),
+ py::arg("phys_origin") = std::array<double, 3>({0., 0., 0.}),
+ py::kw_only(),
+ py::arg("method")=PyGenericIO::FileIO::FileIOPOSIX);
#endif
}
diff --git a/new_python/pygio/__init__.py b/new_python/pygio/__init__.py
index d5707c6a0a3e58df16835037b8ea8667db4001de..17ea2496000907a2e0df98cf6f9f13f8129b320b 100644
--- a/new_python/pygio/__init__.py
+++ b/new_python/pygio/__init__.py
@@ -1,9 +1,20 @@
from __future__ import print_function
-from . import pygio as _pygio
+import os
-try:
- _pygio._init_mpi()
-except:
- print("WARNING: The pygio module has been compiled without MPI support. Writing GenericIO files not supported.")
+_GENERICIO_NO_MPI = False
+if 'GENERICIO_NO_MPI' in os.environ:
+ _GENERICIO_NO_MPI = os.environ['GENERICIO_NO_MPI'].lower() in ['true', 'yes', 'y']
-from .pygio import *
\ No newline at end of file
+if _GENERICIO_NO_MPI:
+ print("WARNING: the pygio module without MPI support has been loaded (due to the GENERICIO_NO_MPI env variable). Writing GenericIO files not supported.")
+ from .pygio_nompi import *
+
+else:
+ # try to load the MPI library (or the no-mpi library, in case of missing MPI during compilation)
+ from . import pygio as _pygio
+ try:
+ _pygio._init_mpi()
+ except:
+ print("WARNING: The pygio module has been compiled without MPI support. Writing GenericIO files not supported.")
+
+ from .pygio import *
\ No newline at end of file