From b929240ec46e1d1f1e5789eb475601b60ba281a9 Mon Sep 17 00:00:00 2001 From: taoliankuan <467460833@qq.com> Date: Thu, 24 Oct 2024 09:55:45 +0800 Subject: [PATCH] support transparent page compression for zstd --- storage/innobase/clone/clone0desc.cc | 6 ++++++ storage/innobase/fil/fil0fil.cc | 1 + storage/innobase/handler/ha_innodb.cc | 11 +++++++++++ storage/innobase/include/os0file.h | 6 +++++- storage/innobase/os/file.cc | 17 +++++++++++++++++ storage/innobase/os/file.h | 9 ++++++++- storage/innobase/os/os0file.cc | 15 +++++++++++++++ 7 files changed, 63 insertions(+), 2 deletions(-) diff --git a/storage/innobase/clone/clone0desc.cc b/storage/innobase/clone/clone0desc.cc index d4c2e4e62..fef80a514 100644 --- a/storage/innobase/clone/clone0desc.cc +++ b/storage/innobase/clone/clone0desc.cc @@ -624,6 +624,8 @@ static const uint CLONE_DESC_FILE_FLAG_ZLIB = 1; static const uint CLONE_DESC_FILE_FLAG_LZ4 = 2; /** Clone File Flag: Encryption type AES */ static const uint CLONE_DESC_FILE_FLAG_AES = 3; +/** Clone File Flag: Encryption type ZSTD */ +static const uint CLONE_DESC_FILE_FLAG_ZSTD = 4; /** File Metadata: Tablespace ID in 4 bytes */ static const uint CLONE_FILE_SPACE_ID_OFFSET = CLONE_FILE_FLAGS_OFFSET + 2; @@ -685,6 +687,8 @@ void Clone_Desc_File_MetaData::serialize(byte *&desc_file, uint &len, DESC_SET_FLAG(file_flags, CLONE_DESC_FILE_FLAG_ZLIB); } else if (m_file_meta.m_compress_type == Compression::LZ4) { DESC_SET_FLAG(file_flags, CLONE_DESC_FILE_FLAG_LZ4); + } else if (m_file_meta.m_compress_type == Compression::ZSTD) { + DESC_SET_FLAG(file_flags, CLONE_DESC_FILE_FLAG_ZSTD); } /* Set file encryption type */ if (m_file_meta.m_encrypt_type == Encryption::AES) { @@ -746,6 +750,8 @@ bool Clone_Desc_File_MetaData::deserialize(const byte *desc_file, m_file_meta.m_compress_type = Compression::ZLIB; } else if (DESC_CHECK_FLAG(file_flags, CLONE_DESC_FILE_FLAG_LZ4)) { m_file_meta.m_compress_type = Compression::LZ4; + } else if (DESC_CHECK_FLAG(file_flags, CLONE_DESC_FILE_FLAG_ZSTD)) { + m_file_meta.m_compress_type = Compression::ZSTD; } /* Get file encryption information */ diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 31e63a5e2..fad688adc 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -9208,6 +9208,7 @@ dberr_t fil_set_compression(space_id_t space_id, const char *algorithm) { switch (srv_debug_compress) { case Compression::LZ4: case Compression::ZLIB: + case Compression::ZSTD: case Compression::NONE: compression.m_type = static_cast(srv_debug_compress); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 08c46bbed..79285dec6 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -2412,6 +2412,9 @@ dberr_t Compression::check(const char *algorithm, Compression *compression) { } else if (innobase_strcasecmp(algorithm, "lz4") == 0) { compression->m_type = LZ4; + } else if (innobase_strcasecmp(algorithm, "zstd") == 0) { + compression->m_type = ZSTD; + } else { return (DB_UNSUPPORTED); } @@ -2435,6 +2438,7 @@ bool Compression::validate(const Compression::Type type) { case NONE: case ZLIB: case LZ4: + case ZSTD: break; default: ret = false; @@ -21668,6 +21672,12 @@ static MYSQL_SYSVAR_UINT( ", 1 is fastest, 9 is best compression and default is 6.", nullptr, nullptr, DEFAULT_COMPRESSION_LEVEL, 0, 9, 0); +static MYSQL_SYSVAR_UINT( + page_compression_level_zstd, page_zstd_level, PLUGIN_VAR_RQCMDARG, + "Compression level used for transparent page compression using zstd." + "Higher level higher radio, 1 is fastest, 22 is best compression and default is 3", + nullptr, nullptr, DEFAULT_COMPRESSION_LEVEL_ZSTD, 1, 22, 0); + static MYSQL_SYSVAR_BOOL( log_compressed_pages, page_zip_log_pages, PLUGIN_VAR_OPCMDARG, "Enables/disables the logging of entire compressed page images." @@ -22615,6 +22625,7 @@ static SYS_VAR *innobase_system_variables[] = { MYSQL_SYSVAR(commit_concurrency), MYSQL_SYSVAR(concurrency_tickets), MYSQL_SYSVAR(compression_level), + MYSQL_SYSVAR(page_compression_level_zstd), MYSQL_SYSVAR(data_file_path), MYSQL_SYSVAR(temp_data_file_path), MYSQL_SYSVAR(data_home_dir), diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index 8dd86b7b7..76308f631 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -82,6 +82,10 @@ extern unsigned long long os_fsync_threshold; /** File offset in bytes */ typedef ib_uint64_t os_offset_t; +constexpr uint32_t DEFAULT_COMPRESSION_LEVEL_ZSTD = 3; + +extern uint page_zstd_level; + namespace file { /** Blocks for doing IO, used in the transparent compression and encryption code. */ @@ -97,7 +101,7 @@ struct Block { byte *m_ptr; /** This padding is needed to avoid false sharing. TBD: of what exactly? We can't use alignas because std::vector uses std::allocator which in - C++14 doesn't have to handle overaligned types. (see § 20.7.9.1.5 of N4140 + C++14 doesn't have to handle overaligned types. (see � 20.7.9.1.5 of N4140 draft) */ byte pad[ut::INNODB_CACHE_LINE_SIZE]; std::atomic m_in_use; diff --git a/storage/innobase/os/file.cc b/storage/innobase/os/file.cc index c22b81d50..e977fbc74 100644 --- a/storage/innobase/os/file.cc +++ b/storage/innobase/os/file.cc @@ -50,6 +50,7 @@ external tools. */ #include #include +#include /** Convert to a "string". @param[in] type The compression type @@ -62,6 +63,8 @@ const char *Compression::to_string(Type type) { return ("Zlib"); case LZ4: return ("LZ4"); + case ZSTD: + return ("ZSTD"); } ut_ad(0); @@ -201,6 +204,20 @@ dberr_t Compression::deserialize(bool dblwr_read, byte *src, byte *dst, break; } + case Compression::ZSTD: { + + size_t csize = ZSTD_decompress(dst, header.m_original_size, ptr, header.m_compressed_size); + if (ZSTD_isError(csize)) { + if (allocated) { + ut_free(dst); + } + return (DB_IO_DECOMPRESS_FAIL); + } + ut_ad(stati(csize) <= len); + len = static_cast(csize); + break; + } + case Compression::LZ4: if (dblwr_read) { diff --git a/storage/innobase/os/file.h b/storage/innobase/os/file.h index b4f53e348..7ed8c85ee 100644 --- a/storage/innobase/os/file.h +++ b/storage/innobase/os/file.h @@ -61,7 +61,10 @@ struct Compression { ZLIB = 1, /** Use LZ4 faster variant, usually lower compression. */ - LZ4 = 2 + LZ4 = 2, + + /** Use Zstd */ + ZSTD = 3 }; /** Compressed page meta-data */ @@ -93,6 +96,7 @@ struct Compression { case NONE: case ZLIB: case LZ4: + case ZSTD: break; default: ut_error; @@ -115,6 +119,9 @@ struct Compression { case LZ4: os << "LZ4"; break; + case ZSTD: + os << "ZSTD"; + break; default: os << ""; break; diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 861bff1ff..119775835 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -95,11 +95,15 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #include #include +#include #ifdef UNIV_HOTBACKUP #include #endif /* UNIV_HOTBACKUP */ +/** Compression level for zstd */ +uint page_zstd_level = DEFAULT_COMPRESSION_LEVEL_ZSTD; + /* Flush after each os_fsync_threshold bytes */ unsigned long long os_fsync_threshold = 0; @@ -1420,6 +1424,17 @@ byte *os_file_compress_page(Compression compression, ulint block_size, break; } + case Compression::ZSTD: { + size_t csize = ZSTD_compress(dst + FIL_PAGE_DATA, static_cast(out_len), src + FIL_PAGE_DATA, static_cast(content_len), page_zstd_level); + if (ZSTD_isError(csize)) { + *dst_len = src_len; + return (src); + } + + len = static_cast(csize); + break; + } + case Compression::LZ4: len = LZ4_compress_default(reinterpret_cast(src) + FIL_PAGE_DATA, -- Gitee