From 54094e6f5eb46f8d9d0880bf44fa7f69c5b58ca8 Mon Sep 17 00:00:00 2001
From: Michael Buehlmann <mbuehlmann@anl.gov>
Date: Tue, 19 Sep 2023 14:53:30 -0500
Subject: [PATCH] some improvements to the python interface
---
.gitignore | 4 +++-
pyproject.toml | 5 +++--
python/CMakeLists.txt | 10 +++++-----
python/genericio.cpp | 22 +++++++++++++++++-----
python/pygio/__init__.py | 35 ++++++++++++++++++++++++++---------
setup.py | 6 ++++--
6 files changed, 58 insertions(+), 24 deletions(-)
diff --git a/.gitignore b/.gitignore
index 516768e..a5af990 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,4 +6,6 @@ frontend/
*.egg-info
.vscode
docs/_build
-.cache
\ No newline at end of file
+.cache
+.venv
+python/pygio/_version.py
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index 346bbad..8ed3444 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,7 +1,8 @@
[build-system]
requires = [
- "setuptools>=42",
+ "setuptools>=45",
+ "setuptools_scm[toml]>=6.2",
"wheel",
"cmake>=3.11",
]
-build-backend = "setuptools.build_meta"
\ No newline at end of file
+build-backend = "setuptools.build_meta"
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index ab2153b..f972f03 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -5,7 +5,7 @@ include(FetchContent)
FetchContent_Declare(
pybind11
GIT_REPOSITORY https://github.com/pybind/pybind11.git
- GIT_TAG v2.9.2
+ GIT_TAG v2.11.1
)
FetchContent_GetProperties(pybind11)
@@ -16,10 +16,10 @@ endif()
# the mpi version
if(MPI_FOUND)
-pybind11_add_module(pygio genericio.cpp)
-target_link_libraries(pygio PRIVATE genericio_mpi)
+pybind11_add_module(pygio_impl genericio.cpp)
+target_link_libraries(pygio_impl PRIVATE genericio_mpi)
endif()
# The no-mpi version
-pybind11_add_module(pygio_nompi genericio.cpp)
-target_link_libraries(pygio_nompi PRIVATE genericio)
\ No newline at end of file
+pybind11_add_module(pygio_nompi_impl genericio.cpp)
+target_link_libraries(pygio_nompi_impl PRIVATE genericio)
\ No newline at end of file
diff --git a/python/genericio.cpp b/python/genericio.cpp
index f51c324..9fd9e47 100644
--- a/python/genericio.cpp
+++ b/python/genericio.cpp
@@ -258,10 +258,11 @@ std::map<std::string, py::dtype> read_variable_dtypes(
int64_t read_num_elems(
std::string filename,
PyGenericIO::FileIO method=PyGenericIO::FileIO::FileIOPOSIX,
- PyGenericIO::MismatchBehavior redistribute=PyGenericIO::MismatchBehavior::MismatchRedistribute
+ PyGenericIO::MismatchBehavior redistribute=PyGenericIO::MismatchBehavior::MismatchRedistribute,
+ int eff_rank=-1
) {
PyGenericIO reader(filename, method, redistribute);
- return reader.readNumElems();
+ return reader.readNumElems(eff_rank);
}
int64_t read_total_num_elems(
@@ -325,10 +326,10 @@ void write_genericio(
#ifdef GENERICIO_NO_MPI
-PYBIND11_MODULE(pygio_nompi, m) {
+PYBIND11_MODULE(pygio_nompi_impl, m) {
m.doc() = "genericio python module (no MPI support)";
#else // GENERICIO_NO_MPI
-PYBIND11_MODULE(pygio, m) {
+PYBIND11_MODULE(pygio_impl, m) {
m.doc() = "genericio python module (with MPI support)";
m.def("_init_mpi", [](){
int initialized;
@@ -418,6 +419,11 @@ PYBIND11_MODULE(pygio, m) {
collective_stats: bool
if ``True``, aggregate statistics among reading ranks (if using MPI)
+ eff_rank: int
+ if -1, use the MPI rank of the calling process (or redistribute if
+ redistribute is set to MismatchRedistribute). Otherwise, use the
+ specified rank (if using MPI).
+
Returns
-------
data: Mapping[str, np.ndarray]
@@ -468,6 +474,11 @@ PYBIND11_MODULE(pygio, m) {
redistribute: PyGenericIO.MismatchBehavior
whether to allow mismatching ranks
+ eff_rank: int
+ if -1, use the MPI rank of the calling process (or redistribute if
+ redistribute is set to MismatchRedistribute). Otherwise, use the
+ specified rank (if using MPI).
+
Returns
-------
nlocal: int
@@ -476,7 +487,8 @@ PYBIND11_MODULE(pygio, m) {
py::arg("filename"),
py::kw_only(),
py::arg("method")=PyGenericIO::FileIO::FileIOPOSIX,
- py::arg("redistribute")=PyGenericIO::MismatchBehavior::MismatchRedistribute);
+ py::arg("redistribute")=PyGenericIO::MismatchBehavior::MismatchRedistribute,
+ py::arg("eff_rank")=-1);
m.def("read_total_num_elems", &read_total_num_elems, R"Delim(
Read the total number of objects (the number of objects that would be
diff --git a/python/pygio/__init__.py b/python/pygio/__init__.py
index ce4f4fb..d85acbb 100644
--- a/python/pygio/__init__.py
+++ b/python/pygio/__init__.py
@@ -1,5 +1,6 @@
from __future__ import print_function
import os
+from ._version import __version__
_GENERICIO_NO_MPI = False
if "GENERICIO_NO_MPI" in os.environ:
@@ -7,22 +8,38 @@ if "GENERICIO_NO_MPI" in os.environ:
if _GENERICIO_NO_MPI:
print(
- "WARNING: the pygio module without MPI support has been loaded (due to the GENERICIO_NO_MPI env variable). Writing GenericIO files not supported."
+ "WARNING: the pygio module without MPI support has been loaded (due to the "
+ "GENERICIO_NO_MPI env variable). Writing GenericIO files not supported."
)
- from .pygio_nompi import *
+ from . import pygio_nompi_impl as _pygio_impl
else:
- # try to load the MPI library (or the no-mpi library, in case of missing MPI during compilation)
- from . import pygio as _pygio
-
+ # try to load the MPI library
try:
- _pygio._init_mpi()
- except:
+ from . import pygio_impl as _pygio_impl
+ except ImportError:
print(
- "WARNING: The pygio module has been compiled without MPI support. Writing GenericIO files not supported."
+ "WARNING: The pygio module has been compiled without MPI support. Writing "
+ "GenericIO files not supported."
)
_GENERICIO_NO_MPI = True
- from .pygio import *
+ from . import pygio_nompi_impl as _pygio_impl
+ else:
+ # MPI version successfully loaded -> init MPI
+ _pygio_impl._init_mpi()
+
+PyGenericIO = _pygio_impl.PyGenericIO
+read_genericio = _pygio_impl.read_genericio
+inspect_genericio = _pygio_impl.inspect_genericio
+read_num_elems = _pygio_impl.read_num_elems
+read_total_num_elems = _pygio_impl.read_total_num_elems
+read_variable_names = _pygio_impl.read_variable_names
+read_variable_dtypes = _pygio_impl.read_variable_dtypes
+read_phys_scale = _pygio_impl.read_phys_scale
+read_phys_origin = _pygio_impl.read_phys_origin
+
+if not _GENERICIO_NO_MPI:
+ write_genericio = _pygio_impl.write_genericio
# move some ENUMs and static functions up to the module namespace
diff --git a/setup.py b/setup.py
index c6f3243..1de630a 100644
--- a/setup.py
+++ b/setup.py
@@ -84,8 +84,10 @@ class CMakeBuild(build_ext):
setup(
name="pygio",
- # version=versioneer.get_version(),
- version=0.1,
+ use_scm_version={
+ "write_to": "python/pygio/_version.py",
+ "write_to_template": '__version__ = "{version}"\n',
+ },
author="",
author_email="",
ext_package="pygio",
--
GitLab