From 6461f57b867b48882e8b02d0c5345cd11ae8b795 Mon Sep 17 00:00:00 2001
From: Hal Finkel <hfinkel@anl.gov>
Date: Fri, 15 Jun 2018 15:18:49 -0500
Subject: [PATCH] start working on sz intrgration
---
GNUmakefile | 48 ++++++++++++++++++++++++++++++++++++--
GenericIO.cxx | 4 ++++
GenericIO.h | 64 ++++++++++++++++++++++++++++++++++++++-------------
3 files changed, 98 insertions(+), 18 deletions(-)
diff --git a/GNUmakefile b/GNUmakefile
index 3fea854..1aaa223 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -57,7 +57,8 @@ BLOSC_CPPFLAGS := \
-Ithirdparty/blosc/internal-complibs/zstd-0.7.4/compress \
-Ithirdparty/blosc/internal-complibs/zstd-0.7.4/common \
-Ithirdparty/blosc/internal-complibs/zstd-0.7.4/dictBuilder \
- -Ithirdparty/blosc/internal-complibs/zstd-0.7.4/decompress
+ -Ithirdparty/blosc/internal-complibs/zstd-0.7.4/decompress \
+ -Ithirdparty/SZ/sz/include
BASE_CPPFLAGS := $(BLOSC_CPPFLAGS) -I. -D__STDC_CONSTANT_MACROS
@@ -125,7 +126,50 @@ BLOSC_O := \
thirdparty/blosc/internal-complibs/snappy-1.1.1/snappy-c.o \
thirdparty/blosc/internal-complibs/snappy-1.1.1/snappy.o \
thirdparty/blosc/internal-complibs/snappy-1.1.1/snappy-sinksource.o \
- thirdparty/blosc/internal-complibs/snappy-1.1.1/snappy-stubs-internal.o
+ thirdparty/blosc/internal-complibs/snappy-1.1.1/snappy-stubs-internal.o \
+ thirdparty/SZ/sz/src/ByteToolkit.o \
+ thirdparty/SZ/sz/src/dataCompression.o \
+ thirdparty/SZ/sz/src/DynamicIntArray.o \
+ thirdparty/SZ/sz/src/iniparser.o \
+ thirdparty/SZ/sz/src/CompressElement.o \
+ thirdparty/SZ/sz/src/DynamicByteArray.o \
+ thirdparty/SZ/sz/src/rw.o \
+ thirdparty/SZ/sz/src/TightDataPointStorageI.o \
+ thirdparty/SZ/sz/src/TightDataPointStorageD.o \
+ thirdparty/SZ/sz/src/TightDataPointStorageF.o \
+ thirdparty/SZ/sz/src/conf.o \
+ thirdparty/SZ/sz/src/DynamicDoubleArray.o \
+ thirdparty/SZ/sz/src/TypeManager.o \
+ thirdparty/SZ/sz/src/dictionary.o \
+ thirdparty/SZ/sz/src/DynamicFloatArray.o \
+ thirdparty/SZ/sz/src/VarSet.o \
+ thirdparty/SZ/sz/src/callZlib.o \
+ thirdparty/SZ/sz/src/Huffman.o \
+ thirdparty/SZ/sz/src/sz_float.o \
+ thirdparty/SZ/sz/src/sz_double.o \
+ thirdparty/SZ/sz/src/sz_int8.o \
+ thirdparty/SZ/sz/src/sz_int16.o \
+ thirdparty/SZ/sz/src/sz_int32.o \
+ thirdparty/SZ/sz/src/sz_int64.o \
+ thirdparty/SZ/sz/src/sz_uint8.o \
+ thirdparty/SZ/sz/src/sz_uint16.o \
+ thirdparty/SZ/sz/src/sz_uint32.o \
+ thirdparty/SZ/sz/src/sz_uint64.o \
+ thirdparty/SZ/sz/src/szd_uint8.o \
+ thirdparty/SZ/sz/src/szd_uint16.o \
+ thirdparty/SZ/sz/src/szd_uint32.o \
+ thirdparty/SZ/sz/src/szd_uint64.o \
+ thirdparty/SZ/sz/src/szd_float.o \
+ thirdparty/SZ/sz/src/szd_double.o \
+ thirdparty/SZ/sz/src/szd_int8.o \
+ thirdparty/SZ/sz/src/szd_int16.o \
+ thirdparty/SZ/sz/src/szd_int32.o \
+ thirdparty/SZ/sz/src/szd_int64.o \
+ thirdparty/SZ/sz/src/sz.o \
+ thirdparty/SZ/sz/src/sz_float_pwr.o \
+ thirdparty/SZ/sz/src/sz_double_pwr.o \
+ thirdparty/SZ/sz/src/szd_float_pwr.o \
+ thirdparty/SZ/sz/src/szd_double_pwr.o
FE_BLOSC_O := $(addprefix $(FEDIR)/,$(BLOSC_O))
diff --git a/GenericIO.cxx b/GenericIO.cxx
index b539017..c76b63b 100644
--- a/GenericIO.cxx
+++ b/GenericIO.cxx
@@ -46,6 +46,7 @@
extern "C" {
#include "blosc.h"
}
+#include "sz.h"
#include <sstream>
#include <fstream>
@@ -351,6 +352,8 @@ struct CompressHeader {
};
const char *CompressName = "BLOSC";
+const char *LossyCompressName = "SZ";
+
#pragma pack()
unsigned GenericIO::DefaultFileIOType = FileIOPOSIX;
@@ -362,6 +365,7 @@ std::size_t GenericIO::CollectiveMPIIOThreshold = 0;
#endif
static bool blosc_initialized = false;
+static bool sz_initialized = false;
#ifndef GENERICIO_NO_MPI
void GenericIO::write() {
diff --git a/GenericIO.h b/GenericIO.h
index 817e701..74c22ac 100644
--- a/GenericIO.h
+++ b/GenericIO.h
@@ -191,6 +191,26 @@ public:
};
public:
+ struct LossyCompressionInfo {
+ enum LCMode {
+ LCModeNone,
+ LCModeAbs,
+ LCModeRel,
+ LCModeAbsAndRel,
+ LCModeAbsOrRel,
+ LCModePSNR
+ };
+
+ LCMode Mode;
+ double AbsErrThreshold;
+ double RelErrThreshold;
+ double PSNRThreshold;
+
+ LossyCompressionInfo()
+ : Mode(LCModeNone), AbsErrThreshold(0.0),
+ RelErrThreshold(0.0), PSNRThreshold(0.0) {}
+ };
+
class Variable {
private:
template <typename ET>
@@ -231,28 +251,33 @@ public:
public:
template <typename T>
- Variable(const std::string &N, T* D, unsigned Flags = 0)
+ Variable(const std::string &N, T* D, unsigned Flags = 0,
+ const LossyCompressionInfo &LCI = LossyCompressionInfo())
: Name(N), Data((void *) D), HasExtraSpace(Flags & VarHasExtraSpace),
IsPhysCoordX(Flags & VarIsPhysCoordX),
IsPhysCoordY(Flags & VarIsPhysCoordY),
IsPhysCoordZ(Flags & VarIsPhysCoordZ),
- MaybePhysGhost(Flags & VarMaybePhysGhost) {
+ MaybePhysGhost(Flags & VarMaybePhysGhost),
+ LCI(LCI) {
deduceTypeInfo(D);
}
template <typename T>
Variable(const std::string &N, std::size_t NumElements, T* D,
- unsigned Flags = 0)
+ unsigned Flags = 0,
+ const LossyCompressionInfo &LCI = LossyCompressionInfo())
: Name(N), Data((void *) D), HasExtraSpace(Flags & VarHasExtraSpace),
IsPhysCoordX(Flags & VarIsPhysCoordX),
IsPhysCoordY(Flags & VarIsPhysCoordY),
IsPhysCoordZ(Flags & VarIsPhysCoordZ),
- MaybePhysGhost(Flags & VarMaybePhysGhost) {
+ MaybePhysGhost(Flags & VarMaybePhysGhost),
+ LCI(LCI) {
deduceTypeInfoFromElement(D);
Size = ElementSize*NumElements;
}
- Variable(const VariableInfo &VI, void *D, unsigned Flags = 0)
+ Variable(const VariableInfo &VI, void *D, unsigned Flags = 0,
+ const LossyCompressionInfo &LCI = LossyCompressionInfo())
: Name(VI.Name), Size(VI.Size), IsFloat(VI.IsFloat),
IsSigned(VI.IsSigned), Data(D),
HasExtraSpace(Flags & VarHasExtraSpace),
@@ -260,7 +285,7 @@ public:
IsPhysCoordY((Flags & VarIsPhysCoordY) || VI.IsPhysCoordY),
IsPhysCoordZ((Flags & VarIsPhysCoordZ) || VI.IsPhysCoordZ),
MaybePhysGhost((Flags & VarMaybePhysGhost) || VI.MaybePhysGhost),
- ElementSize(VI.ElementSize) {}
+ ElementSize(VI.ElementSize), LCI(LCI) {}
std::string Name;
std::size_t Size;
@@ -271,6 +296,8 @@ public:
bool IsPhysCoordX, IsPhysCoordY, IsPhysCoordZ;
bool MaybePhysGhost;
std::size_t ElementSize;
+
+ LossyCompressionInfo LCI;
};
public:
@@ -340,33 +367,38 @@ public:
template <typename T>
void addVariable(const std::string &Name, T *Data,
- unsigned Flags = 0) {
- Vars.push_back(Variable(Name, Data, Flags));
+ unsigned Flags = 0,
+ const LossyCompressionInfo &LCI = LossyCompressionInfo()) {
+ Vars.push_back(Variable(Name, Data, Flags, LCI));
}
template <typename T, typename A>
void addVariable(const std::string &Name, std::vector<T, A> &Data,
- unsigned Flags = 0) {
+ unsigned Flags = 0,
+ const LossyCompressionInfo &LCI = LossyCompressionInfo()) {
T *D = Data.empty() ? 0 : &Data[0];
- addVariable(Name, D, Flags);
+ addVariable(Name, D, Flags, LCI);
}
void addVariable(const VariableInfo &VI, void *Data,
- unsigned Flags = 0) {
- Vars.push_back(Variable(VI, Data, Flags));
+ unsigned Flags = 0,
+ const LossyCompressionInfo &LCI = LossyCompressionInfo()) {
+ Vars.push_back(Variable(VI, Data, Flags, LCI));
}
template <typename T>
void addScalarizedVariable(const std::string &Name, T *Data,
- std::size_t NumElements, unsigned Flags = 0) {
- Vars.push_back(Variable(Name, NumElements, Data, Flags));
+ std::size_t NumElements, unsigned Flags = 0,
+ const LossyCompressionInfo &LCI = LossyCompressionInfo()) {
+ Vars.push_back(Variable(Name, NumElements, Data, Flags, LCI));
}
template <typename T, typename A>
void addScalarizedVariable(const std::string &Name, std::vector<T, A> &Data,
- std::size_t NumElements, unsigned Flags = 0) {
+ std::size_t NumElements, unsigned Flags = 0,
+ const LossyCompressionInfo &LCI = LossyCompressionInfo()) {
T *D = Data.empty() ? 0 : &Data[0];
- addScalarizedVariable(Name, D, NumElements, Flags);
+ addScalarizedVariable(Name, D, NumElements, Flags, LCI);
}
#ifndef GENERICIO_NO_MPI
--
GitLab