From eeacdada6c98ac9491cb69d0397aca70f0c22c54 Mon Sep 17 00:00:00 2001
From: Hal Finkel <hfinkel@anl.gov>
Date: Fri, 28 Sep 2018 12:44:11 -0500
Subject: [PATCH] initial SZ integration - it compiles
---
GenericIO.cxx | 138 ++++++++++++++++++++++++++++++++++++++++++++++----
GenericIO.h | 12 +++++
2 files changed, 140 insertions(+), 10 deletions(-)
diff --git a/GenericIO.cxx b/GenericIO.cxx
index 14f030f..2f297b5 100644
--- a/GenericIO.cxx
+++ b/GenericIO.cxx
@@ -367,6 +367,31 @@ std::size_t GenericIO::CollectiveMPIIOThreshold = 0;
static bool blosc_initialized = false;
static bool sz_initialized = false;
+static int GetSZDT(GenericIO::Variable &Var) {
+ if (Var.hasElementType<float>())
+ return SZ_FLOAT;
+ else if (Var.hasElementType<double>())
+ return SZ_DOUBLE;
+ else if (Var.hasElementType<uint8_t>())
+ return SZ_UINT8;
+ else if (Var.hasElementType<int8_t>())
+ return SZ_INT8;
+ else if (Var.hasElementType<uint16_t>())
+ return SZ_UINT16;
+ else if (Var.hasElementType<int16_t>())
+ return SZ_INT16;
+ else if (Var.hasElementType<uint32_t>())
+ return SZ_UINT32;
+ else if (Var.hasElementType<int32_t>())
+ return SZ_INT32;
+ else if (Var.hasElementType<uint64_t>())
+ return SZ_UINT64;
+ else if (Var.hasElementType<int64_t>())
+ return SZ_INT64;
+ else
+ return -1;
+}
+
#ifndef GENERICIO_NO_MPI
void GenericIO::write() {
if (isBigEndian())
@@ -500,10 +525,60 @@ void GenericIO::write() {
// calculated by the header-writing rank).
memset(&LocalBlockHeaders[i], 0, sizeof(BlockHeader<IsBigEndian>));
if (ShouldCompress) {
+ void *OrigData = Vars[i].Data;
+ bool FreeOrigData = false;
+ size_t OrigUnitSize = Vars[i].Size;
+ size_t OrigDataSize = NElems*Vars[i].Size;
+
+ int FilterIdx = 0;
+ if (Vars[i].LCI.Mode != LossyCompressionInfo::LCModeNone) {
+ int SZDT = GetSZDT(Vars[i]);
+ if (SZDT == -1)
+ goto nosz;
+
+ int EBM;
+ switch (Vars[i].LCI.Mode) {
+ case LossyCompressionInfo::LCModeAbs:
+ EBM = ABS;
+ break;
+ case LossyCompressionInfo::LCModeRel:
+ EBM = REL;
+ break;
+ case LossyCompressionInfo::LCModeAbsAndRel:
+ EBM = ABS_AND_REL;
+ break;
+ case LossyCompressionInfo::LCModeAbsOrRel:
+ EBM = ABS_OR_REL;
+ break;
+ case LossyCompressionInfo::LCModePSNR:
+ EBM = PSNR;
+ break;
+ }
+
+ size_t LOutSize;
+ unsigned char *LCompressedData = SZ_compress_args(SZDT, Vars[i].Data, &LOutSize, EBM,
+ Vars[i].LCI.AbsErrThreshold, Vars[i].LCI.RelErrThreshold,
+ Vars[i].LCI.PSNRThreshold, 0, 0, 0, 0, NElems);
+ if (!LCompressedData)
+ goto nosz;
+ if (LOutSize >= NElems*Vars[i].Size) {
+ free(LCompressedData);
+ goto nosz;
+ }
+
+ OrigData = LCompressedData;
+ FreeOrigData = true;
+ OrigUnitSize = 1;
+ OrigDataSize = LOutSize;
+
+ strncpy(LocalBlockHeaders[i].Filters[FilterIdx++], LossyCompressName, FilterNameSize);
+ }
+nosz:
+
LocalCData[i].resize(sizeof(CompressHeader<IsBigEndian>));
CompressHeader<IsBigEndian> *CH = (CompressHeader<IsBigEndian>*) &LocalCData[i][0];
- CH->OrigCRC = crc64_omp(Vars[i].Data, Vars[i].Size*NElems);
+ CH->OrigCRC = crc64_omp(OrigData, OrigDataSize);
#ifdef _OPENMP
#pragma omp master
@@ -525,13 +600,20 @@ void GenericIO::write() {
}
#endif
- LocalCData[i].resize(LocalCData[i].size() + NElems*Vars[i].Size);
- if (blosc_compress(9, 1, Vars[i].Size, NElems*Vars[i].Size, Vars[i].Data,
+ LocalCData[i].resize(LocalCData[i].size() + OrigDataSize);
+ if (blosc_compress(9, 1, OrigUnitSize, OrigDataSize, OrigData,
&LocalCData[i][0] + sizeof(CompressHeader<IsBigEndian>),
- NElems*Vars[i].Size) <= 0)
+ OrigDataSize) <= 0) {
+ if (FreeOrigData)
+ free(OrigData);
+
goto nocomp;
+ }
- strncpy(LocalBlockHeaders[i].Filters[0], CompressName, FilterNameSize);
+ if (FreeOrigData)
+ free(OrigData);
+
+ strncpy(LocalBlockHeaders[i].Filters[FilterIdx++], CompressName, FilterNameSize);
size_t CNBytes, CCBytes, CBlockSize;
blosc_cbuffer_sizes(&LocalCData[i][0] + sizeof(CompressHeader<IsBigEndian>),
&CNBytes, &CCBytes, &CBlockSize);
@@ -1434,6 +1516,7 @@ void GenericIO::readData(int EffRank, size_t RowOffset, int Rank,
void *VarData = ((char *) Vars[i].Data) + VarOffset;
vector<unsigned char> LData;
+ bool HasSZ = false;
void *Data = VarData;
bool HasExtraSpace = Vars[i].HasExtraSpace;
if (offsetof_safe(GH, BlocksStart) < GH->GlobalHeaderSize &&
@@ -1444,11 +1527,18 @@ void GenericIO::readData(int EffRank, size_t RowOffset, int Rank,
ReadSize = BH->Size + CRCSize;
Offset = BH->Start;
- if (strncmp(BH->Filters[0], CompressName, FilterNameSize) == 0) {
+ int FilterIdx = 0;
+
+ if (strncmp(BH->Filters[FilterIdx], LossyCompressName, FilterNameSize) == 0) {
+ ++FilterIdx;
+ HasSZ = true;
+ }
+
+ if (strncmp(BH->Filters[FilterIdx], CompressName, FilterNameSize) == 0) {
LData.resize(ReadSize);
Data = &LData[0];
HasExtraSpace = true;
- } else if (BH->Filters[0][0] != '\0') {
+ } else if (BH->Filters[FilterIdx][0] != '\0') {
stringstream ss;
ss << "Unknown filter \"" << BH->Filters[0] << "\" on variable " << Vars[i].Name;
throw runtime_error(ss.str());
@@ -1573,22 +1663,50 @@ void GenericIO::readData(int EffRank, size_t RowOffset, int Rank,
blosc_initialized = true;
}
+ if (!sz_initialized) {
+ SZ_Init(NULL);
+ sz_initialized = true;
+ }
+
#ifdef _OPENMP
blosc_set_nthreads(omp_get_max_threads());
}
#endif
+ void *OrigData = VarData;
+ size_t OrigDataSize = Vars[i].Size*RH->NElems;
+
+ if (HasSZ) {
+ size_t CNBytes, CCBytes, CBlockSize;
+ blosc_cbuffer_sizes(&LData[0] + sizeof(CompressHeader<IsBigEndian>),
+ &CNBytes, &CCBytes, &CBlockSize);
+
+ OrigData = malloc(CNBytes);
+ OrigDataSize = CNBytes;
+ }
+
blosc_decompress(&LData[0] + sizeof(CompressHeader<IsBigEndian>),
- VarData, Vars[i].Size*RH->NElems);
+ OrigData, OrigDataSize);
- if (CH->OrigCRC != crc64_omp(VarData, Vars[i].Size*RH->NElems)) {
+ if (CH->OrigCRC != crc64_omp(OrigData, OrigDataSize)) {
++NErrs[2];
break;
}
+
+ if (HasSZ) {
+ int SZDT = GetSZDT(Vars[i]);
+ size_t LDSz = SZ_decompress_args(SZDT, (unsigned char *)OrigData, OrigDataSize,
+ VarData, 0, 0, 0, 0, RH->NElems);
+ free(OrigData);
+
+ if (LDSz != Vars[i].Size*RH->NElems)
+ throw runtime_error("Variable " + Vars[i].Name +
+ ": SZ decompression yielded the wrong amount of data");
+ }
}
// Byte swap the data if necessary.
- if (IsBigEndian != isBigEndian())
+ if (IsBigEndian != isBigEndian() && !HasSZ)
for (size_t j = 0;
j < RH->NElems*(Vars[i].Size/Vars[i].ElementSize); ++j) {
char *Offset = ((char *) VarData) + j*Vars[i].ElementSize;
diff --git a/GenericIO.h b/GenericIO.h
index 74c22ac..ce14f59 100644
--- a/GenericIO.h
+++ b/GenericIO.h
@@ -287,6 +287,18 @@ public:
MaybePhysGhost((Flags & VarMaybePhysGhost) || VI.MaybePhysGhost),
ElementSize(VI.ElementSize), LCI(LCI) {}
+ template <typename ET>
+ bool hasElementType() {
+ if (ElementSize != sizeof(ET))
+ return false;
+ if (IsFloat != !std::numeric_limits<ET>::is_integer)
+ return false;
+ if (IsSigned != std::numeric_limits<ET>::is_signed)
+ return false;
+
+ return true;
+ }
+
std::string Name;
std::size_t Size;
bool IsFloat;
--
GitLab