From 03ba282331f737c4021aa266a360287972e3320f Mon Sep 17 00:00:00 2001
From: Hal Finkel <hfinkel@anl.gov>
Date: Fri, 28 Sep 2018 16:49:07 -0500
Subject: [PATCH] add utility.c
---
GNUmakefile | 1 +
thirdparty/SZ/sz/src/utility.c | 226 +++++++++++++++++++++++++++++++++
2 files changed, 227 insertions(+)
create mode 100644 thirdparty/SZ/sz/src/utility.c
diff --git a/GNUmakefile b/GNUmakefile
index cb95faf..5a0d6c0 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -170,6 +170,7 @@ BLOSC_O := \
thirdparty/SZ/sz/src/szd_int16.o \
thirdparty/SZ/sz/src/szd_int32.o \
thirdparty/SZ/sz/src/szd_int64.o \
+ thirdparty/SZ/sz/src/utility.o \
thirdparty/SZ/sz/src/sz.o \
thirdparty/SZ/sz/src/sz_float_pwr.o \
thirdparty/SZ/sz/src/sz_double_pwr.o \
diff --git a/thirdparty/SZ/sz/src/utility.c b/thirdparty/SZ/sz/src/utility.c
new file mode 100644
index 0000000..be7b02e
--- /dev/null
+++ b/thirdparty/SZ/sz/src/utility.c
@@ -0,0 +1,226 @@
+/**
+ * @file utility.c
+ * @author Sheng Di, Sihuan Li
+ * @date Aug, 2018
+ * @brief
+ * (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
+ * See COPYRIGHT in top-level directory.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include "utility.h"
+#include "sz.h"
+#include "callZlib.h"
+#include "zstd.h"
+
+int compare_struct(const void* obj1, const void* obj2){
+ struct sort_ast_particle * srt1 = (struct sort_ast_particle*)obj1;
+ struct sort_ast_particle * srt2 = (struct sort_ast_particle*)obj2;
+ return srt1->id - srt2->id;
+}
+
+void reorder_vars(SZ_VarSet* vset){
+ SZ_Variable* v[7];
+ SZ_Variable* v_tmp;
+ int i, j;
+ //v[0]
+ for (v_tmp = vset->header->next, i = 0; i < 7; i++){
+ v[i] = v_tmp;
+ v_tmp = v_tmp->next;
+ }
+ //printf("here");
+ size_t dataLen = computeDataLength(v[0]->r5, v[0]->r4, v[0]->r3, v[0]->r2, v[0]->r1);
+ //sihuan debug
+ //printf("the data length is (in sorting): %u", dataLen);
+ struct sort_ast_particle* particle = (struct sort_ast_particle*) malloc(sizeof(struct sort_ast_particle)*dataLen);
+
+ for (i = 0; i < dataLen; i++){
+ particle[i].id = ((int64_t*)v[6]->data)[i];
+ // printf("%llu ", particle[i].id);
+ for (j = 0; j < 6; j++)
+ particle[i].var[j] = ((float*)v[j]->data)[i];
+ }
+
+ //sihuan debug
+ #if 0
+ printf("index before sorting: \n");
+ for (i = 0; i < 5; i++){
+ printf("%llu ", particle[i].id);
+ printf("%.5f ", ((float*)v[0]->data)[i]);
+ }
+ #endif
+ //printf("\n");
+ //sihuan debug
+ //for (i = 0; i < 5; i++)//{
+ //for (j = 0; j < 6; j++)
+ // printf("%.5f ", particle[i].var[j]);
+ // printf("%llu ", particle[i].id );
+ ///}
+ //printf("\n\n");
+
+
+ qsort(particle, dataLen, sizeof(struct sort_ast_particle), compare_struct);
+ for (i = 0; i < dataLen; i++){
+ ((int64_t*)v[6]->data)[i] = particle[i].id;
+ for (j = 0; j < 6; j++)
+ ((float*)v[j]->data)[i] = particle[i].var[j];
+ }
+ free(particle);
+
+ //sihuan debug
+ #if 0
+ for (i = 0; i < 5; i++){
+ printf("%llu ", particle[i].id);
+ printf("%.5f ", ((float*)v[0]->data)[i]);
+ }
+ printf("\n");
+ #endif
+}
+
+size_t intersectAndsort(int64_t* preIndex, size_t preLen, SZ_VarSet* curVar, size_t dataLen, unsigned char* bitarray){
+ size_t i, j, k, m, cnt;
+ i = j = k = m = cnt = 0;
+ SZ_Variable* v[7];
+ SZ_Variable* v_tmp;
+ //v[0]
+ for (v_tmp = curVar->header->next, i = 0; i < 7; i++){
+ v[i] = v_tmp;
+ v_tmp = v_tmp->next;
+ }
+ for (i = 0; i < preLen; i++)
+ bitarray[i] = '0';
+ i = 0;
+ while(i < preLen && j < dataLen){
+ if (preIndex[i] == ((int64_t*)v[6]->data)[j]){
+ cnt++;
+ int64_t tmp;
+ tmp = ((int64_t*)v[6]->data)[k];
+ ((int64_t*)v[6]->data)[k] = ((int64_t*)v[6]->data)[j];
+ ((int64_t*)v[6]->data)[j] = tmp;
+ float data_tmp;
+ for (m = 0; m < 6; m++){
+ data_tmp = ((float*)v[m]->data)[k];
+ ((float*)v[m]->data)[k] = ((float*)v[m]->data)[j];
+ ((float*)v[m]->data)[j] = data_tmp;
+ }
+ k++; i++; j++;
+ }
+ else if (preIndex[i] < ((int64_t*)v[6]->data)[j]){
+ bitarray[i] = '1';
+ i++;
+ }
+ else j++;
+ }
+ printf("intersect count is: %zu, i j k pre curlen is: %zu, %zu, %zu, %zu, %zu\n\n", cnt, i, j, k, preLen, dataLen);
+ return cnt;
+}
+
+void write_reordered_tofile(SZ_VarSet* curVar, size_t dataLen){
+ int var_index; //0 for x, 1 for y...,3 for vx...5 for vz
+ int i;
+ char outputfile_name[256];
+ SZ_Variable* v[7]; SZ_Variable* v_tmp;
+ for (v_tmp = curVar->header->next, i = 0; i < 6; i++){
+ v[i] = v_tmp;
+ v_tmp = v_tmp->next;
+ }
+ for (var_index = 0; var_index < 6; var_index++){
+ sprintf(outputfile_name, "reordered_input_%d_%d.in", sz_tsc->currentStep, var_index);
+ int status_tmp;
+ writeFloatData_inBytes((float*)v[var_index]->data, dataLen, outputfile_name, &status_tmp);
+ }
+}
+
+float calculate_delta_t(size_t size){
+ SZ_Variable* v_tmp = sz_varset->header->next;
+ while(strcmp(v_tmp->varName, "x")) v_tmp = v_tmp->next;
+ float* x1 = (float*) v_tmp->data;
+ float* x0 = (float*) v_tmp->multisteps->hist_data;
+ while(strcmp(v_tmp->varName, "vx")) v_tmp = v_tmp->next;
+ float* vx0 = (float*) v_tmp->multisteps->hist_data;
+ int i, j;
+ double denom = 0.0;
+ double div = 0.0;
+ for (i = 0, j = 0; i < size; i++, j++){
+ while(sz_tsc->bit_array[j] == '1') j++;
+ denom += vx0[j] * (x1[i] - x0[j]);
+ div += vx0[j] * vx0[j];
+ }
+ printf("the calculated delta_t is: %.10f\n", denom/div);
+ return denom/div;
+}
+
+int is_lossless_compressed_data(unsigned char* compressedBytes, size_t cmpSize)
+{
+ int frameContentSize = ZSTD_getFrameContentSize(compressedBytes, cmpSize);
+ if(frameContentSize != ZSTD_CONTENTSIZE_ERROR)
+ return ZSTD_COMPRESSOR;
+
+ int flag = isZlibFormat(compressedBytes[0], compressedBytes[1]);
+ if(flag)
+ return GZIP_COMPRESSOR;
+
+ return -1; //fast mode (without GZIP or ZSTD)
+}
+
+unsigned long sz_lossless_compress(int losslessCompressor, int level, unsigned char* data, unsigned long dataLength, unsigned char** compressBytes)
+{
+ unsigned long outSize = 0;
+ size_t estimatedCompressedSize = 0;
+ switch(losslessCompressor)
+ {
+ case GZIP_COMPRESSOR:
+ outSize = zlib_compress5(data, dataLength, compressBytes, level);
+ break;
+ case ZSTD_COMPRESSOR:
+ estimatedCompressedSize = dataLength*1.2;
+ *compressBytes = (unsigned char*)malloc(estimatedCompressedSize);
+ outSize = ZSTD_compress(*compressBytes, estimatedCompressedSize, data, dataLength, level); //default setting of level is 3
+ break;
+ default:
+ printf("Error: Unrecognized lossless compressor in sz_lossless_compress()\n");
+ }
+ return outSize;
+}
+
+unsigned long sz_lossless_decompress(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData, unsigned long targetOriSize)
+{
+ unsigned long outSize = 0;
+ switch(losslessCompressor)
+ {
+ case GZIP_COMPRESSOR:
+ outSize = zlib_uncompress5(compressBytes, cmpSize, oriData, targetOriSize);
+ break;
+ case ZSTD_COMPRESSOR:
+ *oriData = (unsigned char*)malloc(targetOriSize);
+ ZSTD_decompress(*oriData, targetOriSize, compressBytes, cmpSize);
+ outSize = targetOriSize;
+ break;
+ default:
+ printf("Error: Unrecognized lossless compressor in sz_lossless_decompress()\n");
+ }
+ return outSize;
+}
+
+unsigned long sz_lossless_decompress65536bytes(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData)
+{
+ unsigned long outSize = 0;
+ switch(losslessCompressor)
+ {
+ case GZIP_COMPRESSOR:
+ outSize = zlib_uncompress65536bytes(compressBytes, cmpSize, oriData);
+ break;
+ case ZSTD_COMPRESSOR:
+ *oriData = (unsigned char*)malloc(65536);
+ memset(*oriData, 0, 65536);
+ ZSTD_decompress(*oriData, 65536, compressBytes, cmpSize); //the first 32768 bytes should be exact the same.
+ outSize = 65536;
+ break;
+ default:
+ printf("Error: Unrecognized lossless compressor\n");
+ }
+ return outSize;
+}
--
GitLab