From fc4fc2fce15a81f7f1729a5a6fe7f1ef2678077b Mon Sep 17 00:00:00 2001
From: Hal Finkel <hfinkel@anl.gov>
Date: Fri, 26 Aug 2016 20:02:28 -0500
Subject: [PATCH] add a GIO rewriter (to compress files, etc.)
---
GNUmakefile | 5 ++-
GenericIORewrite.cxx | 79 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 83 insertions(+), 1 deletion(-)
create mode 100644 GenericIORewrite.cxx
diff --git a/GNUmakefile b/GNUmakefile
index 24abb26..3182a7d 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -180,10 +180,13 @@ $(MPIDIR)/GenericIOBenchmarkRead: $(MPIDIR)/GenericIOBenchmarkRead.o $(MPIDIR)/G
$(MPIDIR)/GenericIOBenchmarkWrite: $(MPIDIR)/GenericIOBenchmarkWrite.o $(MPIDIR)/GenericIO.o $(MPI_BLOSC_O)
$(MPICXX) $(MPI_CFLAGS) -o $@ $^
+$(MPIDIR)/GenericIORewrite: $(MPIDIR)/GenericIORewrite.o $(MPIDIR)/GenericIO.o $(MPI_BLOSC_O)
+ $(MPICXX) $(MPI_CFLAGS) -o $@ $^
+
frontend-progs: $(FEDIR)/GenericIOPrint $(FEDIR)/GenericIOVerify
fe-progs: frontend-progs
-mpi-progs: $(MPIDIR)/GenericIOPrint $(MPIDIR)/GenericIOVerify $(MPIDIR)/GenericIOBenchmarkRead $(MPIDIR)/GenericIOBenchmarkWrite
+mpi-progs: $(MPIDIR)/GenericIOPrint $(MPIDIR)/GenericIOVerify $(MPIDIR)/GenericIOBenchmarkRead $(MPIDIR)/GenericIOBenchmarkWrite $(MPIDIR)/GenericIORewrite
frontend-sqlite: $(FEDIR)/GenericIOSQLite.so $(FEDIR)/sqlite3
fe-sqlite: frontend-sqlite
diff --git a/GenericIORewrite.cxx b/GenericIORewrite.cxx
new file mode 100644
index 0000000..ea48348
--- /dev/null
+++ b/GenericIORewrite.cxx
@@ -0,0 +1,79 @@
+#include <cstdlib>
+#include <iostream>
+#include <iomanip>
+#include <string>
+#include <algorithm>
+#include <limits>
+#include <stdexcept>
+#include <stdint.h>
+
+#include "GenericIO.h"
+
+using namespace gio;
+using namespace std;
+
+int main(int argc, char *argv[]) {
+ MPI_Init(&argc, &argv);
+
+ if (argc < 2) {
+ cerr << "Usage: " << argv[0] << " <mpiioOld> <mpiioNew>" << endl;
+ exit(-1);
+ }
+
+ GenericIO::setNaturalDefaultPartition();
+ GenericIO::setDefaultShouldCompress(true);
+
+ {
+ int arg = 1;
+ int Rank, NRanks;
+ MPI_Comm_rank(MPI_COMM_WORLD, &Rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &NRanks);
+
+ string FileName(argv[arg++]);
+ string NewFileName(argv[arg++]);
+
+ unsigned Method = GenericIO::FileIOPOSIX;
+ const char *EnvStr = getenv("GENERICIO_USE_MPIIO");
+ if (EnvStr && string(EnvStr) == "1")
+ Method = GenericIO::FileIOMPI;
+
+ GenericIO GIO(MPI_COMM_WORLD, FileName, Method);
+ GIO.openAndReadHeader(GenericIO::MismatchDisallowed);
+
+ vector<GenericIO::VariableInfo> VI;
+ GIO.getVariableInfo(VI);
+
+ size_t NElem = GIO.readNumElems();
+
+ double PhysOrigin[3], PhysScale[3];
+ GIO.readPhysOrigin(PhysOrigin);
+ GIO.readPhysScale(PhysScale);
+
+ vector< vector<char> > Vars(VI.size());
+ for (size_t i = 0; i < VI.size(); ++i) {
+ Vars[i].resize(VI[i].Size*NElem + GIO.requestedExtraSpace());
+ GIO.addVariable(VI[i], &Vars[i][0], GenericIO::VarHasExtraSpace);
+ }
+
+ GIO.readData(-1, false);
+
+ GenericIO NewGIO(MPI_COMM_WORLD, NewFileName);
+ NewGIO.setNumElems(NElem);
+
+ for (int d = 0; d < 3; ++d) {
+ NewGIO.setPhysOrigin(PhysOrigin[d], d);
+ NewGIO.setPhysScale(PhysScale[d], d);
+ }
+
+ for (size_t i = 0; i < VI.size(); ++i)
+ NewGIO.addVariable(VI[i], &Vars[i][0], GenericIO::VarHasExtraSpace);
+
+ NewGIO.write();
+ }
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ MPI_Finalize();
+
+ return 0;
+}
+
--
GitLab