diff --git a/Makefile b/Makefile index 7d384258aef9a81fa9cd221d5387d65cba398b2f..db6aa14252efd02bc142a829584b3021e5f7dd2e 100644 --- a/Makefile +++ b/Makefile @@ -88,8 +88,8 @@ endif CFLAGS = $(MOREFLAGS) $(CODE_FLAGS) $(OPT_FLAGS_O3) $(DEFINES) CFLAGS_O2 = $(MOREFLAGS) $(CODE_FLAGS) $(OPT_FLAGS_O2) $(DEFINES) -LDFLAGS += $(MOREFLAGS) - +LDFLAGS += $(MOREFLAGS) -lz +LDFLAGS += -lzstd LZO_FILES = lzo/lzo1.o lzo/lzo1a.o lzo/lzo1a_99.o lzo/lzo1b_1.o lzo/lzo1b_2.o lzo/lzo1b_3.o lzo/lzo1b_4.o lzo/lzo1b_5.o LZO_FILES += lzo/lzo1b_6.o lzo/lzo1b_7.o lzo/lzo1b_8.o lzo/lzo1b_9.o lzo/lzo1b_99.o lzo/lzo1b_9x.o lzo/lzo1b_cc.o @@ -106,9 +106,10 @@ UCL_FILES = ucl/alloc.o ucl/n2b_99.o ucl/n2b_d.o ucl/n2b_ds.o ucl/n2b_to.o ucl/n UCL_FILES += ucl/n2d_to.o ucl/n2e_99.o ucl/n2e_d.o ucl/n2e_ds.o ucl/n2e_to.o ucl/ucl_crc.o ucl/ucl_init.o UCL_FILES += ucl/ucl_ptr.o ucl/ucl_str.o ucl/ucl_util.o -ZLIB_FILES = zlib/adler32.o zlib/compress.o zlib/crc32.o zlib/deflate.o zlib/gzclose.o zlib/gzlib.o zlib/gzread.o -ZLIB_FILES += zlib/gzwrite.o zlib/infback.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o zlib/trees.o -ZLIB_FILES += zlib/uncompr.o zlib/zutil.o +#use libz.so +# ZLIB_FILES = zlib/adler32.o zlib/compress.o zlib/crc32.o zlib/deflate.o zlib/gzclose.o zlib/gzlib.o zlib/gzread.o +# ZLIB_FILES += zlib/gzwrite.o zlib/infback.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o zlib/trees.o +# ZLIB_FILES += zlib/uncompr.o zlib/zutil.o LZMAT_FILES = lzmat/lzmat_dec.o lzmat/lzmat_enc.o @@ -131,36 +132,37 @@ BROTLI_FILES += brotli/enc/entropy_encode.o brotli/enc/fast_log.o brotli/enc/his BROTLI_FILES += brotli/enc/metablock.o brotli/enc/static_dict.o brotli/enc/utf8_util.o brotli/enc/compress_fragment.o brotli/enc/compress_fragment_two_pass.o BROTLI_FILES += brotli/enc/cluster.o brotli/enc/bit_cost.o brotli/enc/backward_references_hq.o brotli/enc/dictionary_hash.o -ZSTD_FILES = zstd/lib/common/zstd_common.o -ZSTD_FILES += zstd/lib/common/fse_decompress.o -ZSTD_FILES += zstd/lib/common/xxhash.o -ZSTD_FILES += zstd/lib/common/error_private.o -ZSTD_FILES += zstd/lib/common/entropy_common.o -ZSTD_FILES += zstd/lib/common/pool.o -ZSTD_FILES += zstd/lib/common/debug.o -ZSTD_FILES += zstd/lib/common/threading.o -ZSTD_FILES += zstd/lib/compress/zstd_compress.o -ZSTD_FILES += zstd/lib/compress/zstd_compress_literals.o -ZSTD_FILES += zstd/lib/compress/zstd_compress_sequences.o -ZSTD_FILES += zstd/lib/compress/zstd_compress_superblock.o -ZSTD_FILES += zstd/lib/compress/zstdmt_compress.o -ZSTD_FILES += zstd/lib/compress/zstd_double_fast.o -ZSTD_FILES += zstd/lib/compress/zstd_fast.o -ZSTD_FILES += zstd/lib/compress/zstd_lazy.o -ZSTD_FILES += zstd/lib/compress/zstd_ldm.o -ZSTD_FILES += zstd/lib/compress/zstd_opt.o -ZSTD_FILES += zstd/lib/compress/fse_compress.o -ZSTD_FILES += zstd/lib/compress/huf_compress.o -ZSTD_FILES += zstd/lib/compress/hist.o -ZSTD_FILES += zstd/lib/decompress/zstd_decompress.o -ZSTD_FILES += zstd/lib/decompress/huf_decompress.o -ZSTD_FILES += zstd/lib/decompress/zstd_ddict.o -ZSTD_FILES += zstd/lib/decompress/huf_decompress_amd64.S -ZSTD_FILES += zstd/lib/decompress/zstd_decompress_block.o -ZSTD_FILES += zstd/lib/dictBuilder/cover.o -ZSTD_FILES += zstd/lib/dictBuilder/divsufsort.o -ZSTD_FILES += zstd/lib/dictBuilder/fastcover.o -ZSTD_FILES += zstd/lib/dictBuilder/zdict.o +#use libzstd.so +# ZSTD_FILES = zstd/lib/common/zstd_common.o +# ZSTD_FILES += zstd/lib/common/fse_decompress.o +# ZSTD_FILES += zstd/lib/common/xxhash.o +# ZSTD_FILES += zstd/lib/common/error_private.o +# ZSTD_FILES += zstd/lib/common/entropy_common.o +# ZSTD_FILES += zstd/lib/common/pool.o +# ZSTD_FILES += zstd/lib/common/debug.o +# ZSTD_FILES += zstd/lib/common/threading.o +# ZSTD_FILES += zstd/lib/compress/zstd_compress.o +# ZSTD_FILES += zstd/lib/compress/zstd_compress_literals.o +# ZSTD_FILES += zstd/lib/compress/zstd_compress_sequences.o +# ZSTD_FILES += zstd/lib/compress/zstd_compress_superblock.o +# ZSTD_FILES += zstd/lib/compress/zstdmt_compress.o +# ZSTD_FILES += zstd/lib/compress/zstd_double_fast.o +# ZSTD_FILES += zstd/lib/compress/zstd_fast.o +# ZSTD_FILES += zstd/lib/compress/zstd_lazy.o +# ZSTD_FILES += zstd/lib/compress/zstd_ldm.o +# ZSTD_FILES += zstd/lib/compress/zstd_opt.o +# ZSTD_FILES += zstd/lib/compress/fse_compress.o +# ZSTD_FILES += zstd/lib/compress/huf_compress.o +# ZSTD_FILES += zstd/lib/compress/hist.o +# ZSTD_FILES += zstd/lib/decompress/zstd_decompress.o +# ZSTD_FILES += zstd/lib/decompress/huf_decompress.o +# ZSTD_FILES += zstd/lib/decompress/zstd_ddict.o +# ZSTD_FILES += zstd/lib/decompress/huf_decompress_amd64.S +# ZSTD_FILES += zstd/lib/decompress/zstd_decompress_block.o +# ZSTD_FILES += zstd/lib/dictBuilder/cover.o +# ZSTD_FILES += zstd/lib/dictBuilder/divsufsort.o +# ZSTD_FILES += zstd/lib/dictBuilder/fastcover.o +# ZSTD_FILES += zstd/lib/dictBuilder/zdict.o BRIEFLZ_FILES = brieflz/brieflz.o brieflz/depack.o brieflz/depacks.o @@ -193,11 +195,10 @@ else SNAPPY_FILES = snappy/snappy-sinksource.o snappy/snappy-stubs-internal.o snappy/snappy.o endif -ifeq "$(DONT_BUILD_FASTLZMA2)" "1" +ifeq "$(DONT_BUILD_FASTLZMA2)" "1" #DONT_BUILD_FASTLZMA2=1 DEFINES += -DBENCH_REMOVE_FASTLZMA2 else - FASTLZMA2_SRC = $(wildcard fast-lzma2/*.c) - FASTLZMA2_OBJ = $(FASTLZMA2_SRC:.c=.o) + DEFINES += -DBENCH_REMOVE_FASTLZMA2 endif ifeq "$(DONT_BUILD_XPACK)" "1" @@ -353,7 +354,10 @@ $(NVCOMP_CPP_OBJ): %.cpp.o: %.cpp _lzbench/lzbench.o: _lzbench/lzbench.cpp _lzbench/lzbench.h -lzbench: $(BZIP2_FILES) $(DENSITY_FILES) $(FASTLZMA2_OBJ) $(ZSTD_FILES) $(GLZA_FILES) $(LZSSE_FILES) $(LZFSE_FILES) $(XPACK_FILES) $(GIPFELI_FILES) $(XZ_FILES) $(LIBLZG_FILES) $(BRIEFLZ_FILES) $(LZF_FILES) $(LZRW_FILES) $(BROTLI_FILES) $(CSC_FILES) $(LZMA_FILES) $(ZLING_FILES) $(QUICKLZ_FILES) $(SNAPPY_FILES) $(ZLIB_FILES) $(LZHAM_FILES) $(LZO_FILES) $(UCL_FILES) $(LZMAT_FILES) $(LZ4_FILES) $(LIBDEFLATE_FILES) $(MISC_FILES) $(NVCOMP_FILES) $(LZBENCH_FILES) +# new +lzbench: $(BZIP2_FILES) $(DENSITY_FILES) $(FASTLZMA2_OBJ) $(GLZA_FILES) $(LZSSE_FILES) $(LZFSE_FILES) $(XPACK_FILES) $(GIPFELI_FILES) $(XZ_FILES) $(LIBLZG_FILES) $(BRIEFLZ_FILES) $(LZF_FILES) $(LZRW_FILES) $(BROTLI_FILES) $(CSC_FILES) $(LZMA_FILES) $(ZLING_FILES) $(QUICKLZ_FILES) $(SNAPPY_FILES) $(LZHAM_FILES) $(LZO_FILES) $(UCL_FILES) $(LZMAT_FILES) $(LZ4_FILES) $(LIBDEFLATE_FILES) $(MISC_FILES) $(NVCOMP_FILES) $(LZBENCH_FILES) +#old +# lzbench: $(BZIP2_FILES) $(DENSITY_FILES) $(FASTLZMA2_OBJ) $(ZSTD_FILES) $(GLZA_FILES) $(LZSSE_FILES) $(LZFSE_FILES) $(XPACK_FILES) $(GIPFELI_FILES) $(XZ_FILES) $(LIBLZG_FILES) $(BRIEFLZ_FILES) $(LZF_FILES) $(LZRW_FILES) $(BROTLI_FILES) $(CSC_FILES) $(LZMA_FILES) $(ZLING_FILES) $(QUICKLZ_FILES) $(SNAPPY_FILES) $(ZLIB_FILES) $(LZHAM_FILES) $(LZO_FILES) $(UCL_FILES) $(LZMAT_FILES) $(LZ4_FILES) $(LIBDEFLATE_FILES) $(MISC_FILES) $(NVCOMP_FILES) $(LZBENCH_FILES) $(CXX) $^ -o $@ $(LDFLAGS) @echo Linked GCC_VERSION=$(GCC_VERSION) CLANG_VERSION=$(CLANG_VERSION) COMPILER=$(COMPILER) diff --git a/_lzbench/compressors.cpp b/_lzbench/compressors.cpp index e50a3f92163fc3609c75d4ad21ff93f18e945486..741b49482156e961d58f8511c2d71b5eb67bab57 100644 --- a/_lzbench/compressors.cpp +++ b/_lzbench/compressors.cpp @@ -1744,6 +1744,120 @@ int64_t lzbench_zstd_LDM_compress(char *inbuf, size_t insize, char *outbuf, size ZSTD_CCtx_setParameter(zstd_params->cctx, ZSTD_c_enableLongDistanceMatching, 1); return lzbench_zstd_compress(inbuf, insize, outbuf, outsize, level, windowLog, (char*) zstd_params); } + +size_t ZstdStreamCompressBuff(ZSTD_CStream *cStream, uint8_t* dst, uint32_t dstSize, + uint8_t *src, size_t srcSize, size_t buffInSize, size_t buffOutSize) +{ + size_t ret; + uint32_t lastChunk = 0; + void *buffIn = malloc(buffInSize); + void *buffOut = malloc(buffOutSize); + + uint32_t cSize = 0; + uint32_t srcPos = 0; + uint32_t leftSize = srcSize; + uint32_t readCount = 0; + + while (!lastChunk) { + readCount = (buffInSize <= leftSize) ? buffInSize : leftSize; + lastChunk = readCount < buffInSize; + + memcpy(buffIn, src + srcPos, readCount); + srcPos += readCount; + leftSize -= readCount; + + ZSTD_inBuffer input = { buffIn, readCount, 0 }; + ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; + while (input.pos != input.size) { + ret = ZSTD_compressStream(cStream, &output, &input); + + memcpy(dst + cSize, buffOut, output.pos); + cSize += output.pos; + output.pos = 0; + } + } + + ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; + do { + leftSize = ZSTD_endStream(cStream, &output); + memcpy(dst + cSize, buffOut, output.pos); + cSize += output.pos; + output.pos = 0; + } while (leftSize != 0); + + free(buffIn); + free(buffOut); + + return cSize; +} + +int64_t lzbench_zstd_stream_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t, char*) +{ + ZSTD_CStream *zstdCStream = ZSTD_createCStream(); + if (level != 0) { + ZSTD_initCStream(zstdCStream, level); //JD mm场景没有次接口调用逻辑,按level作为默认等级场景 + } + uint32_t cSize = ZstdStreamCompressBuff(zstdCStream, (uint8_t *)outbuf, outsize, (uint8_t *)inbuf, insize, 1024, 1024); + ZSTD_freeCStream(zstdCStream); + return (int64_t) cSize; +} + +uint32_t ZstdStreamDecompressBuff(ZSTD_DStream *dStream, uint8_t *src, size_t srcSize, uint8_t *dst, size_t cSize, + size_t buffInSize, size_t buffOutSize) +{ + size_t ret; + void *dBuffIn = malloc(buffInSize); + void *dBuffOut = malloc(buffOutSize); + + uint32_t lastChunk = 0, dSize = 0, dstPos = 0, leftSize = cSize; + + while (!lastChunk) { + size_t readCount = (buffInSize <= leftSize) ? buffInSize : leftSize; + + lastChunk = readCount < buffInSize; + + memcpy(dBuffIn, dst + dstPos, readCount); + dstPos += readCount; + + size_t cSize = 1; + + ZSTD_inBuffer dInput = { dBuffIn, readCount, 0 }; + ZSTD_outBuffer dOutput = { dBuffOut, buffOutSize, 0 }; + size_t size = 1; + while (dInput.pos != dInput.size) { + size = ZSTD_decompressStream(dStream, &dOutput, &dInput); + + if (dOutput.pos != 0) { + memcpy(src + dSize, dBuffOut, dOutput.pos); + } + + dSize += dOutput.pos; + dOutput.pos = 0; + } + + if (size == 0) { + break; + } + + + leftSize -= readCount; + } + + free(dBuffOut); + free(dBuffIn); + return dSize; +} + +int64_t lzbench_zstd_stream_decompress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*) +{ + + ZSTD_DStream *zstdDStream = ZSTD_createDStream(); + ZSTD_initDStream(zstdDStream); + + uint32_t dSize = ZstdStreamDecompressBuff(zstdDStream, (uint8_t *)outbuf, outsize, (uint8_t *)inbuf, insize, 1024, 1024); + ZSTD_freeDStream(zstdDStream); + return dSize; +} #endif diff --git a/_lzbench/compressors.h b/_lzbench/compressors.h index 5f0bfa1cfb145da5e0d02024d81cd0810461a51c..a4fa6bc5baed4893323725a1e80edacad4c5d6a3 100644 --- a/_lzbench/compressors.h +++ b/_lzbench/compressors.h @@ -478,6 +478,8 @@ int64_t lzbench_return_0(char *inbuf, size_t insize, char *outbuf, size_t outsiz int64_t lzbench_zstd_decompress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*); char* lzbench_zstd_LDM_init(size_t insize, size_t level, size_t); int64_t lzbench_zstd_LDM_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*); + int64_t lzbench_zstd_stream_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t, char*); + int64_t lzbench_zstd_stream_decompress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*); #else #define lzbench_zstd_init NULL #define lzbench_zstd_deinit NULL diff --git a/_lzbench/lzbench.h b/_lzbench/lzbench.h index 1b33da47fd5cbd4d0ea91b9a3f6761c08a2fbd46..8a8c89599613523251dbb09e3711c4871e4d0c51 100644 --- a/_lzbench/lzbench.h +++ b/_lzbench/lzbench.h @@ -136,7 +136,7 @@ typedef struct -#define LZBENCH_COMPRESSOR_COUNT 72 +#define LZBENCH_COMPRESSOR_COUNT 73 static const compressor_desc_t comp_desc[LZBENCH_COMPRESSOR_COUNT] = { @@ -209,6 +209,7 @@ static const compressor_desc_t comp_desc[LZBENCH_COMPRESSOR_COUNT] = { "zstdLDM", "1.5.5", 1, 22, 0, 0, lzbench_zstd_LDM_compress, lzbench_zstd_decompress, lzbench_zstd_LDM_init, lzbench_zstd_deinit }, { "zstd22LDM", "1.5.5", 1, 22, 22, 0, lzbench_zstd_LDM_compress, lzbench_zstd_decompress, lzbench_zstd_LDM_init, lzbench_zstd_deinit }, { "zstd24LDM", "1.5.5", 1, 22, 24, 0, lzbench_zstd_LDM_compress, lzbench_zstd_decompress, lzbench_zstd_LDM_init, lzbench_zstd_deinit }, + { "zstd_stream", "1.5.2", 1, 4, 0, 0, lzbench_zstd_stream_compress, lzbench_zstd_stream_decompress, NULL, NULL }, { "nakamichi", "okamigan", 0, 0, 0, 0, lzbench_nakamichi_compress, lzbench_nakamichi_decompress, NULL, NULL }, { "cudaMemcpy", "", 0, 0, 0, 0, lzbench_cuda_return_0, lzbench_cuda_memcpy, lzbench_cuda_init, lzbench_cuda_deinit }, { "nvcomp_lz4", "1.2.2", 0, 5, 0, 0, lzbench_nvcomp_compress, lzbench_nvcomp_decompress, lzbench_nvcomp_init, lzbench_nvcomp_deinit },