Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • hacc/genericio
  • mbuehlmann/genericio
2 results
Show changes
View open merge request
Commits on Source (4)
Hide whitespace changes
Inline Side-by-side
Showing
with 439 additions and 167 deletions
.gitignore
Edit View file @ f695c851
......@@ -8,4 +8,4 @@ frontend/
docs/_build
.cache
.venv
python/pygio/_version.py
\ No newline at end of file
python/pygio/_version.py
GNUmakefile
Edit View file @ f695c851
......@@ -106,6 +106,65 @@ $(FEDIR)/%.o: %.cxx | $(FEDIR)
mkdir -p $(dir $@)
$(CXX) $(FE_CXXFLAGS) $(FE_CPPFLAGS) -c -o $@ $<
SZ_O := \
thirdparty/SZ/sz/src/ArithmeticCoding.o \
thirdparty/SZ/sz/src/ByteToolkit.o \
thirdparty/SZ/sz/src/CacheTable.o \
thirdparty/SZ/sz/src/callZlib.o \
thirdparty/SZ/sz/src/CompressElement.o \
thirdparty/SZ/sz/src/conf.o \
thirdparty/SZ/sz/src/dataCompression.o \
thirdparty/SZ/sz/src/dictionary.o \
thirdparty/SZ/sz/src/DynamicByteArray.o \
thirdparty/SZ/sz/src/DynamicDoubleArray.o \
thirdparty/SZ/sz/src/DynamicFloatArray.o \
thirdparty/SZ/sz/src/DynamicIntArray.o \
thirdparty/SZ/sz/src/exafelSZ.o \
thirdparty/SZ/sz/src/Huffman.o \
thirdparty/SZ/sz/src/iniparser.o \
thirdparty/SZ/sz/src/MultiLevelCacheTable.o \
thirdparty/SZ/sz/src/MultiLevelCacheTableWideInterval.o \
thirdparty/SZ/sz/src/pastri.o \
thirdparty/SZ/sz/src/rw.o \
thirdparty/SZ/sz/src/rwf.o \
thirdparty/SZ/sz/src/sz.o \
thirdparty/SZ/sz/src/szd_double.o \
thirdparty/SZ/sz/src/szd_double_pwr.o \
thirdparty/SZ/sz/src/szd_double_ts.o \
thirdparty/SZ/sz/src/szd_float.o \
thirdparty/SZ/sz/src/szd_float_pwr.o \
thirdparty/SZ/sz/src/szd_float_ts.o \
thirdparty/SZ/sz/src/szd_int16.o \
thirdparty/SZ/sz/src/szd_int32.o \
thirdparty/SZ/sz/src/szd_int64.o \
thirdparty/SZ/sz/src/szd_int8.o \
thirdparty/SZ/sz/src/sz_double.o \
thirdparty/SZ/sz/src/sz_double_pwr.o \
thirdparty/SZ/sz/src/sz_double_ts.o \
thirdparty/SZ/sz/src/szd_uint16.o \
thirdparty/SZ/sz/src/szd_uint32.o \
thirdparty/SZ/sz/src/szd_uint64.o \
thirdparty/SZ/sz/src/szd_uint8.o \
thirdparty/SZ/sz/src/szf.o \
thirdparty/SZ/sz/src/sz_float.o \
thirdparty/SZ/sz/src/sz_float_pwr.o \
thirdparty/SZ/sz/src/sz_float_ts.o \
thirdparty/SZ/sz/src/sz_int16.o \
thirdparty/SZ/sz/src/sz_int32.o \
thirdparty/SZ/sz/src/sz_int64.o \
thirdparty/SZ/sz/src/sz_int8.o \
thirdparty/SZ/sz/src/sz_omp.o \
thirdparty/SZ/sz/src/sz_uint16.o \
thirdparty/SZ/sz/src/sz_uint32.o \
thirdparty/SZ/sz/src/sz_uint64.o \
thirdparty/SZ/sz/src/sz_uint8.o \
thirdparty/SZ/sz/src/TightDataPointStorageD.o \
thirdparty/SZ/sz/src/TightDataPointStorageF.o \
thirdparty/SZ/sz/src/TightDataPointStorageI.o \
thirdparty/SZ/sz/src/TypeManager.o \
thirdparty/SZ/sz/src/utility.o \
thirdparty/SZ/sz/src/VarSet.o
BLOSC_O := \
thirdparty/blosc/blosc.o \
thirdparty/blosc/blosclz.o \
......@@ -152,54 +211,8 @@ BLOSC_O := \
thirdparty/blosc/internal-complibs/snappy-1.1.1/snappy.o \
thirdparty/blosc/internal-complibs/snappy-1.1.1/snappy-sinksource.o \
thirdparty/blosc/internal-complibs/snappy-1.1.1/snappy-stubs-internal.o \
thirdparty/SZ/sz/src/ByteToolkit.o \
thirdparty/SZ/sz/src/dataCompression.o \
thirdparty/SZ/sz/src/DynamicIntArray.o \
thirdparty/SZ/sz/src/iniparser.o \
thirdparty/SZ/sz/src/CompressElement.o \
thirdparty/SZ/sz/src/DynamicByteArray.o \
thirdparty/SZ/sz/src/rw.o \
thirdparty/SZ/sz/src/TightDataPointStorageI.o \
thirdparty/SZ/sz/src/TightDataPointStorageD.o \
thirdparty/SZ/sz/src/TightDataPointStorageF.o \
thirdparty/SZ/sz/src/conf.o \
thirdparty/SZ/sz/src/DynamicDoubleArray.o \
thirdparty/SZ/sz/src/TypeManager.o \
thirdparty/SZ/sz/src/dictionary.o \
thirdparty/SZ/sz/src/DynamicFloatArray.o \
thirdparty/SZ/sz/src/VarSet.o \
thirdparty/SZ/sz/src/callZlib.o \
thirdparty/SZ/sz/src/Huffman.o \
thirdparty/SZ/sz/src/sz_float.o \
thirdparty/SZ/sz/src/sz_double.o \
thirdparty/SZ/sz/src/sz_int8.o \
thirdparty/SZ/sz/src/sz_int16.o \
thirdparty/SZ/sz/src/sz_int32.o \
thirdparty/SZ/sz/src/sz_int64.o \
thirdparty/SZ/sz/src/sz_uint8.o \
thirdparty/SZ/sz/src/sz_uint16.o \
thirdparty/SZ/sz/src/sz_uint32.o \
thirdparty/SZ/sz/src/sz_uint64.o \
thirdparty/SZ/sz/src/szd_uint8.o \
thirdparty/SZ/sz/src/szd_uint16.o \
thirdparty/SZ/sz/src/szd_uint32.o \
thirdparty/SZ/sz/src/szd_uint64.o \
thirdparty/SZ/sz/src/szd_float.o \
thirdparty/SZ/sz/src/szd_double.o \
thirdparty/SZ/sz/src/szd_int8.o \
thirdparty/SZ/sz/src/szd_int16.o \
thirdparty/SZ/sz/src/szd_int32.o \
thirdparty/SZ/sz/src/szd_int64.o \
thirdparty/SZ/sz/src/utility.o \
thirdparty/SZ/sz/src/sz.o \
thirdparty/SZ/sz/src/sz_float_pwr.o \
thirdparty/SZ/sz/src/sz_double_pwr.o \
thirdparty/SZ/sz/src/szd_float_pwr.o \
thirdparty/SZ/sz/src/szd_double_pwr.o \
thirdparty/SZ/sz/src/sz_double_ts.o \
thirdparty/SZ/sz/src/sz_float_ts.o \
thirdparty/SZ/sz/src/szd_double_ts.o \
thirdparty/SZ/sz/src/szd_float_ts.o
BLOSC_O += ${SZ_O}
ifdef VELOC_INSTALL_DIR
BLOSC_O := $(BLOSC_O) \
......
GenericIO.cxx
Edit View file @ f695c851
......@@ -603,7 +603,7 @@ void GenericIO::write() {
if (SZDT == -1)
goto nosz;
int EBM;
int EBM = 0;
switch (Vars[i].LCI.Mode) {
case LossyCompressionInfo::LCModeAbs:
EBM = ABS;
......@@ -617,15 +617,18 @@ void GenericIO::write() {
case LossyCompressionInfo::LCModeAbsOrRel:
EBM = ABS_OR_REL;
break;
case LossyCompressionInfo::LCModePWRel:
EBM = PW_REL;
break;
case LossyCompressionInfo::LCModePSNR:
EBM = PSNR;
break;
}
size_t LOutSize;
size_t LOutSize = 0;
unsigned char *LCompressedData = SZ_compress_args(SZDT, Vars[i].Data, &LOutSize, EBM,
Vars[i].LCI.AbsErrThreshold, Vars[i].LCI.RelErrThreshold,
Vars[i].LCI.PSNRThreshold, 0, 0, 0, 0, NElems);
Vars[i].LCI.PWRelErrThreshold, 0, 0, 0, 0, NElems);
if (!LCompressedData)
goto nosz;
if (LOutSize >= NElems*Vars[i].Size) {
......
GenericIO.h
Edit View file @ f695c851
......@@ -184,6 +184,7 @@ public:
LCModeNone,
LCModeAbs,
LCModeRel,
LCModePWRel,
LCModeAbsAndRel,
LCModeAbsOrRel,
LCModePSNR
......@@ -192,11 +193,12 @@ public:
LCMode Mode;
double AbsErrThreshold;
double RelErrThreshold;
double PWRelErrThreshold;
double PSNRThreshold;
LossyCompressionInfo()
: Mode(LCModeNone), AbsErrThreshold(0.0),
RelErrThreshold(0.0), PSNRThreshold(0.0) {}
RelErrThreshold(0.0), PWRelErrThreshold(0.0), PSNRThreshold(0.0) {}
};
class Variable {
......
thirdparty/CMakeLists.txt
Edit View file @ f695c851
set(SZ_Sources
SZ/sz/src/ArithmeticCoding.c
SZ/sz/src/ByteToolkit.c
SZ/sz/src/CacheTable.c
SZ/sz/src/callZlib.c
SZ/sz/src/CompressElement.c
SZ/sz/src/conf.c
SZ/sz/src/dataCompression.c
SZ/sz/src/dictionary.c
SZ/sz/src/DynamicByteArray.c
SZ/sz/src/DynamicDoubleArray.c
SZ/sz/src/DynamicFloatArray.c
SZ/sz/src/DynamicIntArray.c
SZ/sz/src/exafelSZ.c
SZ/sz/src/Huffman.c
SZ/sz/src/iniparser.c
SZ/sz/src/MultiLevelCacheTable.c
SZ/sz/src/MultiLevelCacheTableWideInterval.c
SZ/sz/src/pastri.c
SZ/sz/src/rw.c
SZ/sz/src/rwf.c
SZ/sz/src/sz.c
SZ/sz/src/szd_double.c
SZ/sz/src/szd_double_pwr.c
SZ/sz/src/szd_double_ts.c
SZ/sz/src/szd_float.c
SZ/sz/src/szd_float_pwr.c
SZ/sz/src/szd_float_ts.c
SZ/sz/src/szd_int16.c
SZ/sz/src/szd_int32.c
SZ/sz/src/szd_int64.c
SZ/sz/src/szd_int8.c
SZ/sz/src/sz_double.c
SZ/sz/src/sz_double_pwr.c
SZ/sz/src/sz_double_ts.c
SZ/sz/src/szd_uint16.c
SZ/sz/src/szd_uint32.c
SZ/sz/src/szd_uint64.c
SZ/sz/src/szd_uint8.c
SZ/sz/src/szf.c
SZ/sz/src/sz_float.c
SZ/sz/src/sz_float_pwr.c
SZ/sz/src/sz_float_ts.c
SZ/sz/src/sz_int16.c
SZ/sz/src/sz_int32.c
SZ/sz/src/sz_int64.c
SZ/sz/src/sz_int8.c
SZ/sz/src/sz_omp.c
SZ/sz/src/sz_uint16.c
SZ/sz/src/sz_uint32.c
SZ/sz/src/sz_uint64.c
SZ/sz/src/sz_uint8.c
SZ/sz/src/TightDataPointStorageD.c
SZ/sz/src/TightDataPointStorageF.c
SZ/sz/src/TightDataPointStorageI.c
SZ/sz/src/TypeManager.c
SZ/sz/src/utility.c
SZ/sz/src/VarSet.c
)
set(BLOSC_Sources
blosc/blosc.c
blosc/blosclz.c
......@@ -44,55 +104,9 @@ set(BLOSC_Sources
blosc/internal-complibs/snappy-1.1.1/snappy.cxx
blosc/internal-complibs/snappy-1.1.1/snappy-sinksource.cxx
blosc/internal-complibs/snappy-1.1.1/snappy-stubs-internal.cxx
SZ/sz/src/ByteToolkit.c
SZ/sz/src/dataCompression.c
SZ/sz/src/DynamicIntArray.c
SZ/sz/src/iniparser.c
SZ/sz/src/CompressElement.c
SZ/sz/src/DynamicByteArray.c
SZ/sz/src/rw.c
SZ/sz/src/TightDataPointStorageI.c
SZ/sz/src/TightDataPointStorageD.c
SZ/sz/src/TightDataPointStorageF.c
SZ/sz/src/conf.c
SZ/sz/src/DynamicDoubleArray.c
SZ/sz/src/TypeManager.c
SZ/sz/src/dictionary.c
SZ/sz/src/DynamicFloatArray.c
SZ/sz/src/VarSet.c
SZ/sz/src/callZlib.c
SZ/sz/src/Huffman.c
SZ/sz/src/sz_float.c
SZ/sz/src/sz_double.c
SZ/sz/src/sz_int8.c
SZ/sz/src/sz_int16.c
SZ/sz/src/sz_int32.c
SZ/sz/src/sz_int64.c
SZ/sz/src/sz_uint8.c
SZ/sz/src/sz_uint16.c
SZ/sz/src/sz_uint32.c
SZ/sz/src/sz_uint64.c
SZ/sz/src/szd_uint8.c
SZ/sz/src/szd_uint16.c
SZ/sz/src/szd_uint32.c
SZ/sz/src/szd_uint64.c
SZ/sz/src/szd_float.c
SZ/sz/src/szd_double.c
SZ/sz/src/szd_int8.c
SZ/sz/src/szd_int16.c
SZ/sz/src/szd_int32.c
SZ/sz/src/szd_int64.c
SZ/sz/src/utility.c
SZ/sz/src/sz.c
SZ/sz/src/sz_float_pwr.c
SZ/sz/src/sz_double_pwr.c
SZ/sz/src/szd_float_pwr.c
SZ/sz/src/szd_double_pwr.c
SZ/sz/src/sz_double_ts.c
SZ/sz/src/sz_float_ts.c
SZ/sz/src/szd_double_ts.c
SZ/sz/src/szd_float_ts.c
${SZ_Sources}
)
set(BLOSC_include_dirs
blosc
SZ/sz/include
......@@ -106,10 +120,11 @@ set(BLOSC_include_dirs
blosc/internal-complibs/zstd-0.7.4/dictBuilder
blosc/internal-complibs/zstd-0.7.4/decompress
)
add_library(blosc STATIC ${BLOSC_Sources})
set_property(TARGET blosc PROPERTY C_STANDARD 99)
target_include_directories(blosc PRIVATE ${BLOSC_include_dirs})
target_include_directories(blosc INTERFACE
target_include_directories(blosc INTERFACE
blosc
SZ/sz/include
)
......
thirdparty/SZ/sz/include/ArithmeticCoding.h 0 → 100644
Edit View file @ f695c851
/**
* @file ArithmeticCoding.h
* @author Sheng Di
* @date Dec, 2018
* @brief Header file for the ArithmeticCoding.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _ArithmeticCoding_H
#define _ArithmeticCoding_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#define ONE_FOURTH (0x40000000000) //44 bits are absolutely enough to deal with a large dataset (support at most 16TB per process)
#define ONE_HALF (0x80000000000)
#define THREE_FOURTHS (0xC0000000000)
#define MAX_CODE (0xFFFFFFFFFFF)
#define MAX_INTERVALS 1048576 //the limit to the arithmetic coding (at most 2^(20) intervals)
typedef struct Prob {
size_t low;
size_t high;
int state;
} Prob;
typedef struct AriCoder
{
int numOfRealStates; //the # real states menas the number of states after the optimization of # intervals
int numOfValidStates; //the # valid states means the number of non-zero frequency cells (some states/codes actually didn't appear)
size_t total_frequency;
Prob* cumulative_frequency; //used to encode data more efficiencly
} AriCoder;
void output_bit_1(unsigned int* buf);
void output_bit_0(unsigned int* buf);
unsigned int output_bit_1_plus_pending(int pending_bits);
unsigned int output_bit_0_plus_pending(int pending_bits);
AriCoder *createAriCoder(int numOfStates, int *s, size_t length);
void freeAriCoder(AriCoder *ariCoder);
void ari_init(AriCoder *ariCoder, int *s, size_t length);
unsigned int pad_ariCoder(AriCoder* ariCoder, unsigned char** out);
int unpad_ariCoder(AriCoder** ariCoder, unsigned char* bytes);
unsigned char get_bit(unsigned char* p, int offset);
void ari_encode(AriCoder *ariCoder, int *s, size_t length, unsigned char *out, size_t *outSize);
void ari_decode(AriCoder *ariCoder, unsigned char *s, size_t s_len, size_t targetLength, int *out);
Prob* getCode(AriCoder *ariCoder, size_t scaled_value);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _ArithmeticCoding_H ----- */
thirdparty/SZ/sz/include/ByteToolkit.h
Edit View file @ f695c851
......@@ -18,29 +18,29 @@ extern "C" {
//ByteToolkit.c
unsigned short bytesToUInt16_bigEndian(unsigned char* bytes);
unsigned int bytesToUInt32_bigEndian(unsigned char* bytes);
unsigned long bytesToUInt64_bigEndian(unsigned char* b);
extern unsigned short bytesToUInt16_bigEndian(unsigned char* bytes);
extern unsigned int bytesToUInt32_bigEndian(unsigned char* bytes);
extern unsigned long bytesToUInt64_bigEndian(unsigned char* b);
short bytesToInt16_bigEndian(unsigned char* bytes);
int bytesToInt32_bigEndian(unsigned char* bytes);
long bytesToInt64_bigEndian(unsigned char* b);
int bytesToInt_bigEndian(unsigned char* bytes);
extern short bytesToInt16_bigEndian(unsigned char* bytes);
extern int bytesToInt32_bigEndian(unsigned char* bytes);
extern long bytesToInt64_bigEndian(unsigned char* b);
extern int bytesToInt_bigEndian(unsigned char* bytes);
void intToBytes_bigEndian(unsigned char *b, unsigned int num);
extern void intToBytes_bigEndian(unsigned char *b, unsigned int num);
void int64ToBytes_bigEndian(unsigned char *b, uint64_t num);
void int32ToBytes_bigEndian(unsigned char *b, uint32_t num);
void int16ToBytes_bigEndian(unsigned char *b, uint16_t num);
extern void int64ToBytes_bigEndian(unsigned char *b, uint64_t num);
extern void int32ToBytes_bigEndian(unsigned char *b, uint32_t num);
extern void int16ToBytes_bigEndian(unsigned char *b, uint16_t num);
long bytesToLong_bigEndian(unsigned char* b);
void longToBytes_bigEndian(unsigned char *b, unsigned long num);
extern long bytesToLong_bigEndian(unsigned char* b);
extern void longToBytes_bigEndian(unsigned char *b, unsigned long num);
long doubleToOSEndianLong(double value);
int floatToOSEndianInt(float value);
short getExponent_float(float value);
short getPrecisionReqLength_float(float precision);
short getExponent_double(double value);
short getPrecisionReqLength_double(double precision);
extern short getExponent_float(float value);
extern short getPrecisionReqLength_float(float precision);
extern short getExponent_double(double value);
extern short getPrecisionReqLength_double(double precision);
unsigned char numberOfLeadingZeros_Int(int i);
unsigned char numberOfLeadingZeros_Long(long i);
unsigned char getLeadingNumbers_Int(int v1, int v2);
......@@ -49,15 +49,15 @@ short bytesToShort(unsigned char* bytes);
void shortToBytes(unsigned char* b, short value);
int bytesToInt(unsigned char* bytes);
long bytesToLong(unsigned char* bytes);
float bytesToFloat(unsigned char* bytes);
void floatToBytes(unsigned char *b, float num);
double bytesToDouble(unsigned char* bytes);
void doubleToBytes(unsigned char *b, double num);
extern float bytesToFloat(unsigned char* bytes);
extern void floatToBytes(unsigned char *b, float num);
extern double bytesToDouble(unsigned char* bytes);
extern void doubleToBytes(unsigned char *b, double num);
int extractBytes(unsigned char* byteArray, size_t k, int validLength);
int getMaskRightCode(int m);
int getLeftMovingCode(int kMod8);
int getRightMovingSteps(int kMod8, int resiBitLength);
int getRightMovingCode(int kMod8, int resiBitLength);
extern int getLeftMovingCode(int kMod8);
extern int getRightMovingSteps(int kMod8, int resiBitLength);
extern int getRightMovingCode(int kMod8, int resiBitLength);
short* convertByteDataToShortArray(unsigned char* bytes, size_t byteLength);
unsigned short* convertByteDataToUShortArray(unsigned char* bytes, size_t byteLength);
......@@ -68,8 +68,10 @@ void convertUIntArrayToBytes(unsigned int* states, size_t stateLength, unsigned
void convertLongArrayToBytes(int64_t* states, size_t stateLength, unsigned char* bytes);
void convertULongArrayToBytes(uint64_t* states, size_t stateLength, unsigned char* bytes);
size_t bytesToSize(unsigned char* bytes);
void sizeToBytes(unsigned char* outBytes, size_t size);
extern size_t bytesToSize(unsigned char* bytes);
extern void sizeToBytes(unsigned char* outBytes, size_t size);
void put_codes_to_output(unsigned int buf, int bitSize, unsigned char** p, int* lackBits, size_t *outSize);
#ifdef __cplusplus
}
......
thirdparty/SZ/sz/include/CacheTable.h 0 → 100644
Edit View file @ f695c851
/**
* @file CacheTable.h
* @author Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang, Sheng Di, Dingwen Tao
* @date Jan, 2019
* @brief Header file.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef SZ_MASTER_CACHETABLE_H
#define SZ_MASTER_CACHETABLE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "stdio.h"
#include "stdint.h"
#include <math.h>
extern double* g_CacheTable;
extern uint32_t * g_InverseTable;
extern uint32_t baseIndex;
extern uint32_t topIndex;
extern int bits;
int doubleGetExpo(double d);
int CacheTableGetRequiredBits(double precision, int quantization_intervals);
uint32_t CacheTableGetIndex(float value, int bits);
uint64_t CacheTableGetIndexDouble(double value, int bits);
int CacheTableIsInBoundary(uint32_t index);
void CacheTableBuild(double * table, int count, double smallest, double largest, double precision, int quantization_intervals);
uint32_t CacheTableFind(uint32_t index);
void CacheTableFree();
#ifdef __cplusplus
}
#endif
#endif //SZ_MASTER_CACHETABLE_H
thirdparty/SZ/sz/include/CompressElement.h
Edit View file @ f695c851
......@@ -40,32 +40,33 @@ typedef struct LossyCompressionElement
unsigned char integerMidBytes[8];
int integerMidBytes_Length; //they are mid_bits actually
//char curBytes[8];
//int curBytes_Length; //4 for single_precision or 8 for double_precision
//int curBytes_Length; //4 for single_precision or 8 for double_precision
int resMidBitsLength;
int residualMidBits;
} LossyCompressionElement;
char* decompressGroupIDArray(unsigned char* bytes, size_t dataLength);
short computeGroupNum_float(float value);
short computeGroupNum_double(double value);
extern short computeGroupNum_float(float value);
extern short computeGroupNum_double(double value);
void listAdd_double(double last3CmprsData[3], double value);
void listAdd_float(float last3CmprsData[3], float value);
void listAdd_int(int64_t last3CmprsData[3], int64_t value);
void listAdd_float_group(float *groups, int *flags, char groupNum, float oriValue, float decValue, char* curGroupID);
void listAdd_double_group(double *groups, int *flags, char groupNum, double oriValue, double decValue, char* curGroupID);
extern void listAdd_double(double last3CmprsData[3], double value);
extern void listAdd_float(float last3CmprsData[3], float value);
extern void listAdd_int(int64_t last3CmprsData[3], int64_t value);
void listAdd_int32(int32_t last3CmprsData[3], int32_t value);
extern void listAdd_float_group(float *groups, int *flags, char groupNum, float oriValue, float decValue, char* curGroupID);
extern void listAdd_double_group(double *groups, int *flags, char groupNum, double oriValue, double decValue, char* curGroupID);
int validPrediction_double(double minErr, double precision);
int validPrediction_float(float minErr, float precision);
double* generateGroupErrBounds(int errorBoundMode, double realPrecision, double pwrErrBound);
int generateGroupMaxIntervalCount(double* groupErrBounds);
void new_LossyCompressionElement(LossyCompressionElement *lce, int leadingNum, unsigned char* intMidBytes,
void new_LossyCompressionElement(LossyCompressionElement *lce, int leadingNum, unsigned char* intMidBytes,
int intMidBytes_Length, int resiMidBitsLength, int resiBits);
void updateLossyCompElement_Double(unsigned char* curBytes, unsigned char* preBytes,
void updateLossyCompElement_Double(unsigned char* curBytes, unsigned char* preBytes,
int reqBytesLength, int resiBitsLength, LossyCompressionElement *lce);
void updateLossyCompElement_Float(unsigned char* curBytes, unsigned char* preBytes,
extern void updateLossyCompElement_Float(unsigned char* curBytes, unsigned char* preBytes,
int reqBytesLength, int resiBitsLength, LossyCompressionElement *lce);
#ifdef __cplusplus
......
thirdparty/SZ/sz/include/DynamicByteArray.h
Edit View file @ f695c851
......@@ -16,7 +16,7 @@ extern "C" {
#include <stdio.h>
typedef struct DynamicByteArray
{
{
unsigned char* array;
size_t size;
size_t capacity;
......@@ -26,8 +26,8 @@ void new_DBA(DynamicByteArray **dba, size_t cap);
void convertDBAtoBytes(DynamicByteArray *dba, unsigned char** bytes);
void free_DBA(DynamicByteArray *dba);
unsigned char getDBA_Data(DynamicByteArray *dba, size_t pos);
void addDBA_Data(DynamicByteArray *dba, unsigned char value);
void memcpyDBA_Data(DynamicByteArray *dba, unsigned char* data, size_t length);
extern void addDBA_Data(DynamicByteArray *dba, unsigned char value);
extern void memcpyDBA_Data(DynamicByteArray *dba, unsigned char* data, size_t length);
#ifdef __cplusplus
}
......
thirdparty/SZ/sz/include/DynamicIntArray.h
Edit View file @ f695c851
......@@ -16,7 +16,7 @@ extern "C" {
#include <stdio.h>
typedef struct DynamicIntArray
{
{
unsigned char* array; //char* (one byte) is enough, don't have to be int*
size_t size;
size_t capacity;
......@@ -26,7 +26,7 @@ void new_DIA(DynamicIntArray **dia, size_t cap);
void convertDIAtoInts(DynamicIntArray *dia, unsigned char **data);
void free_DIA(DynamicIntArray *dia);
int getDIA_Data(DynamicIntArray *dia, size_t pos);
void addDIA_Data(DynamicIntArray *dia, int value);
extern void addDIA_Data(DynamicIntArray *dia, int value);
#ifdef __cplusplus
}
......
thirdparty/SZ/sz/include/Huffman.h
Edit View file @ f695c851
......@@ -26,8 +26,8 @@ typedef struct node_t {
} *node;
typedef struct HuffmanTree {
int stateNum;
int allNodes;
unsigned int stateNum;
unsigned int allNodes;
struct node_t* pool;
node *qqq, *qq; //the root node of the HuffmanTree is qq[1]
int n_nodes; //n_nodes is for compression
......@@ -35,6 +35,7 @@ typedef struct HuffmanTree {
unsigned long **code;
unsigned char *cout;
int n_inode; //n_inode is for decompression
int maxBitCount;
} HuffmanTree;
HuffmanTree* createHuffmanTree(int stateNum);
......@@ -46,8 +47,12 @@ void qinsert(HuffmanTree *huffmanTree, node n);
node qremove(HuffmanTree *huffmanTree);
void build_code(HuffmanTree *huffmanTree, node n, int len, unsigned long out1, unsigned long out2);
void init(HuffmanTree *huffmanTree, int *s, size_t length);
void init_static(HuffmanTree *huffmanTree, int *s, size_t length);
void encode(HuffmanTree *huffmanTree, int *s, size_t length, unsigned char *out, size_t *outSize);
void decode(unsigned char *s, size_t targetLength, node t, int *out);
void decode_MSST19(unsigned char *s, size_t targetLength, node t, int *out, int maxBits);
void pad_tree_uchar(HuffmanTree* huffmanTree, unsigned char* L, unsigned char* R, unsigned int* C, unsigned char* t, unsigned int i, node root);
void pad_tree_ushort(HuffmanTree* huffmanTree, unsigned short* L, unsigned short* R, unsigned int* C, unsigned char* t, unsigned int i, node root);
void pad_tree_uint(HuffmanTree* huffmanTree, unsigned int* L, unsigned int* R, unsigned int* C, unsigned char* t, unsigned int i, node root);
......@@ -58,8 +63,9 @@ void unpad_tree_uint(HuffmanTree* huffmanTree, unsigned int* L, unsigned int* R,
node reconstruct_HuffTree_from_bytes_anyStates(HuffmanTree *huffmanTree, unsigned char* bytes, int nodeCount);
void encode_withTree(HuffmanTree* huffmanTree, int *s, size_t length, unsigned char **out, size_t *outSize);
int encode_withTree_MSST19(HuffmanTree* huffmanTree, int *s, size_t length, unsigned char **out, size_t *outSize);
void decode_withTree(HuffmanTree* huffmanTree, unsigned char *s, size_t targetLength, int *out);
void decode_withTree_MSST19(HuffmanTree* huffmanTree, unsigned char *s, size_t targetLength, int *out, int maxBits);
void SZ_ReleaseHuffman(HuffmanTree* huffmanTree);
#ifdef __cplusplus
......
thirdparty/SZ/sz/include/MultiLevelCacheTable.h 0 → 100644
Edit View file @ f695c851
/**
* @file MultiLevelCacheTable.h
* @author Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang, Sheng Di, Dingwen Tao
* @date Jan, 2019
* @brief Header file.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _MULTILEVELCACHETABLE_H
#define _MULTILEVELCACHETABLE_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <memory.h>
#include <stdlib.h>
#include "stdio.h"
typedef struct SubLevelTable{
uint32_t baseIndex;
uint32_t topIndex;
uint32_t* table;
uint8_t expoIndex;
} SubLevelTable;
typedef struct TopLevelTable{
uint8_t bits;
uint8_t baseIndex;
uint8_t topIndex;
struct SubLevelTable* subTables;
float bottomBoundary;
float topBoundary;
} TopLevelTable;
uint8_t MLCT_GetExpoIndex(float value);
uint8_t MLCT_GetRequiredBits(float precision);
uint32_t MLCT_GetMantiIndex(float value, int bits);
float MLTC_RebuildFloat(uint8_t expo, uint32_t manti, int bits);
void MultiLevelCacheTableBuild(struct TopLevelTable* topTable, float* precisionTable, int count, float precision);
uint32_t MultiLevelCacheTableGetIndex(float value, struct TopLevelTable* topLevelTable);
void MultiLevelCacheTableFree(struct TopLevelTable* table);
#ifdef __cplusplus
}
#endif
#endif //_MULTILEVELCACHETABLE_H
thirdparty/SZ/sz/include/MultiLevelCacheTableWideInterval.h 0 → 100644
Edit View file @ f695c851
/**
* @file MultiLevelCacheTableWideInterval.h
* @author Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang, Sheng Di, Dingwen Tao
* @date Jan, 2019
* @brief Header file for MultiLevelCacheTableWideInterval.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _MULTILEVELCACHETABLEWIDEINTERVAL_H
#define _MULTILEVELCACHETABLEWIDEINTERVAL_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <memory.h>
#include <stdlib.h>
#include "stdio.h"
typedef struct SubLevelTableWideInterval{
uint64_t baseIndex;
uint64_t topIndex;
uint16_t* table;
uint16_t expoIndex;
} SubLevelTableWideInterval;
typedef struct TopLevelTableWideInterval{
uint16_t bits;
uint16_t baseIndex;
uint16_t topIndex;
struct SubLevelTableWideInterval* subTables;
double bottomBoundary;
double topBoundary;
} TopLevelTableWideInterval;
void freeTopLevelTableWideInterval(struct TopLevelTableWideInterval* topTable);
uint16_t MLCTWI_GetExpoIndex(double value);
uint16_t MLCTWI_GetRequiredBits(double precision);
uint64_t MLCTWI_GetMantiIndex(double value, int bits);
double MLTCWI_RebuildDouble(uint16_t expo, uint64_t manti, int bits);
void MultiLevelCacheTableWideIntervalBuild(struct TopLevelTableWideInterval* topTable, double* precisionTable, int count, double precision, int plus_bits);
uint32_t MultiLevelCacheTableWideIntervalGetIndex(double value, struct TopLevelTableWideInterval* topLevelTable);
void MultiLevelCacheTableWideIntervalFree(struct TopLevelTableWideInterval* table);
#ifdef __cplusplus
}
#endif
#endif //_MULTILEVELCACHETABLEWIDEINTERVAL_H
thirdparty/SZ/sz/include/TightDataPointStorageD.h
Edit View file @ f695c851
......@@ -58,6 +58,9 @@ typedef struct TightDataPointStorageD
unsigned char* raBytes;
size_t raBytes_size;
unsigned char plus_bits;
unsigned char max_bits;
} TightDataPointStorageD;
void new_TightDataPointStorageD_Empty(TightDataPointStorageD **self);
......
thirdparty/SZ/sz/include/TightDataPointStorageF.h
Edit View file @ f695c851
......@@ -59,6 +59,9 @@ typedef struct TightDataPointStorageF
unsigned char* raBytes;
size_t raBytes_size;
unsigned char plus_bits;
unsigned char max_bits;
} TightDataPointStorageF;
......
thirdparty/SZ/sz/include/TightDataPointStorageI.h
Edit View file @ f695c851
......@@ -52,7 +52,7 @@ void new_TightDataPointStorageI(TightDataPointStorageI **self,
int* type, unsigned char* exactDataBytes, size_t exactDataBytes_size,
double realPrecision, long minValue, int intervals, int dataType);
void convertTDPStoBytes_int(TightDataPointStorageI* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte);
void convertTDPStoBytes_int(TightDataPointStorageI* tdps, unsigned char* bytes, unsigned char sameByte);
void convertTDPStoFlatBytes_int(TightDataPointStorageI *tdps, unsigned char** bytes, size_t *size);
void convertTDPStoFlatBytes_int_args(TightDataPointStorageI *tdps, unsigned char* bytes, size_t *size);
void free_TightDataPointStorageI(TightDataPointStorageI *tdps);
......
thirdparty/SZ/sz/include/VarSet.h
Edit View file @ f695c851
......@@ -22,11 +22,14 @@ typedef struct sz_multisteps
int predictionMode;
int lastSnapshotStep; //the previous snapshot step
unsigned int currentStep; //current time step of the execution/simulation
//void* ori_data; //original data pointer, which serve as the key for retrieving hist_data
void* hist_data; //historical data in past time steps
} sz_multisteps;
typedef struct SZ_Variable
{
unsigned char var_id;
char* varName;
char compressType; //102 means HZ; 101 means SZ
int dataType; //SZ_FLOAT or SZ_DOUBLE
......@@ -56,11 +59,13 @@ typedef struct SZ_VarSet
void free_Variable_keepOriginalData(SZ_Variable* v);
void free_Variable_keepCompressedBytes(SZ_Variable* v);
void free_Variable_all(SZ_Variable* v);
void SZ_batchAddVar(char* varName, int dataType, void* data,
void SZ_batchAddVar(int var_id, char* varName, int dataType, void* data,
int errBoundMode, double absErrBound, double relBoundRatio, double pwRelBoundRatio,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
int SZ_batchDelVar_vset(SZ_VarSet* vset, char* varName);
int SZ_batchDelVar(char* varName);
int SZ_batchDelVar_ID_vset(SZ_VarSet* vset, int var_id);
int SZ_batchDelVar_ID(int var_id);
SZ_Variable* SZ_searchVar(char* varName);
void* SZ_getVarData(char* varName, size_t *r5, size_t *r4, size_t *r3, size_t *r2, size_t *r1);
......@@ -69,6 +74,8 @@ void free_VarSet_vset(SZ_VarSet *vset, int mode);
void SZ_freeVarSet(int mode);
void free_multisteps(sz_multisteps* multisteps);
int checkVarID(unsigned char cur_var_id, unsigned char* var_ids, int var_count);
SZ_Variable* SZ_getVariable(int var_id);
#ifdef __cplusplus
}
......
thirdparty/SZ/sz/include/conf.h
Edit View file @ f695c851
......@@ -21,9 +21,13 @@ void updateQuantizationInfo(int quant_intervals);
int SZ_ReadConf(const char* sz_cfgFile);
int SZ_LoadConf(const char* sz_cfgFile);
int checkVersion(char* version);
int computeVersion(int major, int minor, int revision);
int checkVersion2(char* version);
void initSZ_TSC();
unsigned int roundUpToPowerOf2(unsigned int base);
double computeABSErrBoundFromPSNR(double psnr, double threshold, double value_range);
double computeABSErrBoundFromNORM_ERR(double normErr, size_t nbEle);
#ifdef __cplusplus
}
......
thirdparty/SZ/sz/include/dataCompression.h
Edit View file @ f695c851
......@@ -16,6 +16,7 @@ extern "C" {
#include "sz.h"
#include <stdio.h>
#include <stdbool.h>
#define computeMinMax(data) \
for(i=1;i<size;i++)\
......@@ -33,7 +34,10 @@ int computeByteSizePerIntValue(long valueRangeSize);
long computeRangeSize_int(void* oriData, int dataType, size_t size, int64_t* valueRangeSize);
double computeRangeSize_double(double* oriData, size_t size, double* valueRangeSize, double* medianValue);
float computeRangeSize_float(float* oriData, size_t size, float* valueRangeSize, float* medianValue);
float computeRangeSize_double_subblock(double* oriData, double* valueRangeSize, double* medianValue,
float computeRangeSize_float_MSST19(float* oriData, size_t size, float* valueRangeSize, float* medianValue, unsigned char * signs, bool* positive, float* nearZero);
double computeRangeSize_double_MSST19(double* oriData, size_t size, double* valueRangeSize, double* medianValue, unsigned char * signs, bool* positive, double* nearZero);
double computeRangeSize_double_subblock(double* oriData, double* valueRangeSize, double* medianValue,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1);
......@@ -48,27 +52,30 @@ float max_f(float a, float b);
double getRealPrecision_double(double valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status);
double getRealPrecision_float(float valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status);
double getRealPrecision_int(long valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status);
void symTransform_8bytes(unsigned char data[8]);
void symTransform_2bytes(unsigned char data[2]);
void symTransform_4bytes(unsigned char data[4]);
extern void symTransform_8bytes(unsigned char data[8]);
extern void symTransform_2bytes(unsigned char data[2]);
extern void symTransform_4bytes(unsigned char data[4]);
void compressInt8Value(int8_t tgtValue, int8_t minValue, int byteSize, unsigned char* bytes);
void compressInt16Value(int16_t tgtValue, int16_t minValue, int byteSize, unsigned char* bytes);
void compressInt32Value(int32_t tgtValue, int32_t minValue, int byteSize, unsigned char* bytes);
void compressInt64Value(int64_t tgtValue, int64_t minValue, int byteSize, unsigned char* bytes);
extern void compressInt8Value(int8_t tgtValue, int8_t minValue, int byteSize, unsigned char* bytes);
extern void compressInt16Value(int16_t tgtValue, int16_t minValue, int byteSize, unsigned char* bytes);
extern void compressInt32Value(int32_t tgtValue, int32_t minValue, int byteSize, unsigned char* bytes);
extern void compressInt64Value(int64_t tgtValue, int64_t minValue, int byteSize, unsigned char* bytes);
void compressUInt8Value(uint8_t tgtValue, uint8_t minValue, int byteSize, unsigned char* bytes);
void compressUInt16Value(uint16_t tgtValue, uint16_t minValue, int byteSize, unsigned char* bytes);
void compressUInt32Value(uint32_t tgtValue, uint32_t minValue, int byteSize, unsigned char* bytes);
void compressUInt64Value(uint64_t tgtValue, uint64_t minValue, int byteSize, unsigned char* bytes);
extern void compressUInt8Value(uint8_t tgtValue, uint8_t minValue, int byteSize, unsigned char* bytes);
extern void compressUInt16Value(uint16_t tgtValue, uint16_t minValue, int byteSize, unsigned char* bytes);
extern void compressUInt32Value(uint32_t tgtValue, uint32_t minValue, int byteSize, unsigned char* bytes);
extern void compressUInt64Value(uint64_t tgtValue, uint64_t minValue, int byteSize, unsigned char* bytes);
void compressSingleFloatValue(FloatValueCompressElement *vce, float tgtValue, float precision, float medianValue,
extern void compressSingleFloatValue(FloatValueCompressElement *vce, float tgtValue, float precision, float medianValue,
int reqLength, int reqBytesLength, int resiBitsLength);
void compressSingleDoubleValue(DoubleValueCompressElement *vce, double tgtValue, double precision, double medianValue,
void compressSingleFloatValue_MSST19(FloatValueCompressElement *vce, float tgtValue, float precision, int reqLength, int reqBytesLength, int resiBitsLength);
void compressSingleDoubleValue(DoubleValueCompressElement *vce, double tgtValue, double precision, double medianValue,
int reqLength, int reqBytesLength, int resiBitsLength);
void compressSingleDoubleValue_MSST19(DoubleValueCompressElement *vce, double tgtValue, double precision, int reqLength, int reqBytesLength, int resiBitsLength);
int compIdenticalLeadingBytesCount_double(unsigned char* preBytes, unsigned char* curBytes);
int compIdenticalLeadingBytesCount_float(unsigned char* preBytes, unsigned char* curBytes);
void addExactData(DynamicByteArray *exactMidByteArray, DynamicIntArray *exactLeadNumArray,
extern int compIdenticalLeadingBytesCount_float(unsigned char* preBytes, unsigned char* curBytes);
extern void addExactData(DynamicByteArray *exactMidByteArray, DynamicIntArray *exactLeadNumArray,
DynamicIntArray *resiBitArray, LossyCompressionElement *lce);
int getPredictionCoefficients(int layers, int dimension, int **coeff_array, int *status);
......@@ -78,13 +85,13 @@ int computeBlockEdgeSize_2D(int segmentSize);
int initRandomAccessBytes(unsigned char* raBytes);
int generateLossyCoefficients_float(float* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, float* medianValue, float* decData);
int compressExactDataArray_float(float* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,
int compressExactDataArray_float(float* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,
int reqLength, int reqBytesLength, int resiBitsLength, float medianValue);
void decompressExactDataArray_float(unsigned char* leadNum, unsigned char* exactMidBytes, unsigned char* residualMidBits, size_t nbEle, int reqLength, float medianValue, float** decData);
int generateLossyCoefficients_double(double* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, double* medianValue, double* decData);
int compressExactDataArray_double(double* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,
int compressExactDataArray_double(double* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,
int reqLength, int reqBytesLength, int resiBitsLength, double medianValue);
void decompressExactDataArray_double(unsigned char* leadNum, unsigned char* exactMidBytes, unsigned char* residualMidBits, size_t nbEle, int reqLength, double medianValue, double** decData);
......

Virtual Help Desk | CELS Gitlab Instructions | Privacy Policy