From 0ee61fc03b476951e698dadc8ffcb667cfa4021c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A6=99=E8=95=89=E5=82=A8=E8=93=84=E6=89=80?= <727854256@qq.com> Date: Wed, 15 Jan 2025 18:13:16 +0800 Subject: [PATCH] =?UTF-8?q?[add|chg]1.=E6=B7=BB=E5=8A=A0=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E8=A1=A8=E4=B8=AD=E7=9A=84=E8=A7=A3=E6=9E=90=E3=80=81=E4=BF=9D?= =?UTF-8?q?=E5=AD=98=E5=8E=8B=E7=BC=A9=E5=8F=82=E6=95=B0=E6=B5=81=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/access/common/reloptions.c | 8 ++- src/backend/catalog/heap.c | 9 ++- src/backend/catalog/index.c | 3 +- src/backend/storage/page/Makefile | 3 +- src/backend/storage/page/page_compression.c | 54 ++++++++++++++ src/backend/utils/cache/relcache.c | 18 ++++- src/include/catalog/heap.h | 3 +- src/include/storage/page_compression.h | 79 +++++++++++++++++++++ src/include/storage/relfilenode.h | 1 + src/include/utils/rel.h | 18 +++++ src/include/utils/relcache.h | 3 +- 11 files changed, 190 insertions(+), 9 deletions(-) create mode 100644 src/backend/storage/page/page_compression.c create mode 100644 src/include/storage/page_compression.h diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index 5b69604..6527eab 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -1885,7 +1885,13 @@ default_reloptions(Datum reloptions, bool validate, relopt_kind kind) {"vacuum_index_cleanup", RELOPT_TYPE_ENUM, offsetof(StdRdOptions, vacuum_index_cleanup)}, {"vacuum_truncate", RELOPT_TYPE_BOOL, - offsetof(StdRdOptions, vacuum_truncate)} + offsetof(StdRdOptions, vacuum_truncate)}, + {"compresstype", RELOPT_TYPE_INT, offsetof(StdRdOptions, compress) + offsetof(PageCompressOpts, compressType)}, + {"compress_level", RELOPT_TYPE_INT, offsetof(StdRdOptions, compress) + offsetof(PageCompressOpts, compressLevel)}, + {"compress_chunk_size", RELOPT_TYPE_INT, offsetof(StdRdOptions, compress) + offsetof(PageCompressOpts, compressChunkSize)}, + {"compress_prealloc_chunks", RELOPT_TYPE_INT, offsetof(StdRdOptions, compress) + offsetof(PageCompressOpts, compressPreallocChunks)}, + {"compress_byte_convert", RELOPT_TYPE_BOOL, offsetof(StdRdOptions, compress) + offsetof(PageCompressOpts, compressByteConvert)}, + {"compress_diff_convert", RELOPT_TYPE_BOOL, offsetof(StdRdOptions, compress) + offsetof(PageCompressOpts, compressDiffConvert)}, }; return (bytea *) build_reloptions(reloptions, validate, kind, diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index a21de89..d149045 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -299,7 +299,8 @@ heap_create(const char *relname, bool allow_system_table_mods, TransactionId *relfrozenxid, MultiXactId *relminmxid, - bool create_storage) + bool create_storage, + Datum reloptions) { Relation rel; @@ -373,7 +374,8 @@ heap_create(const char *relname, shared_relation, mapped_relation, relpersistence, - relkind); + relkind, + reloptions); /* * Have the storage manager create the relation's disk file, if needed. @@ -1286,7 +1288,8 @@ heap_create_with_catalog(const char *relname, allow_system_table_mods, &relfrozenxid, &relminmxid, - true); + true, + reloptions); Assert(relid == RelationGetRelid(new_rel_desc)); diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index db697a9..d5ea907 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -961,7 +961,8 @@ index_create(Relation heapRelation, allow_system_table_mods, &relfrozenxid, &relminmxid, - create_storage); + create_storage, + (Datum) 0); Assert(relfrozenxid == InvalidTransactionId); Assert(relminmxid == InvalidMultiXactId); diff --git a/src/backend/storage/page/Makefile b/src/backend/storage/page/Makefile index da539b1..3531913 100644 --- a/src/backend/storage/page/Makefile +++ b/src/backend/storage/page/Makefile @@ -15,7 +15,8 @@ include $(top_builddir)/src/Makefile.global OBJS = \ bufpage.o \ checksum.o \ - itemptr.o + itemptr.o \ + page_compression.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/storage/page/page_compression.c b/src/backend/storage/page/page_compression.c new file mode 100644 index 0000000..1e46be7 --- /dev/null +++ b/src/backend/storage/page/page_compression.c @@ -0,0 +1,54 @@ +#include "storage/page_compression.h" +#include "utils/rel.h" + +uint8 ConvertChunkSize(uint32 compressedChunkSize, bool *success) +{ + uint8 chunkSize = INDEX_OF_HALF_BLCKSZ; + switch (compressedChunkSize) { + case BLCKSZ / 2: + chunkSize = INDEX_OF_HALF_BLCKSZ; + case BLCKSZ / 4: + chunkSize = INDEX_OF_QUARTER_BLCKSZ; + case BLCKSZ / 8: + chunkSize = INDEX_OF_EIGHTH_BLCKSZ; + case BLCKSZ / 16: + chunkSize = INDEX_OF_SIXTEENTHS_BLCKSZ; + default: + *success = false; + return chunkSize; + } + *success = true; + return chunkSize; +} + +void SetupPageCompressForRelation(RelFileNode *node, PageCpmpressOpts *compressOptions, const char *relationName) +{ + uint32 algorithm = (uint32)compressOptions->compressType; + if (algorithm == (uint32)COMPRESS_TYPE_NONE) { + node->opt = 0; + } else { + uint8 compressLevel; + bool symbol = false; + if (compressOptions->compressLevel >= 0) { + symbol = true; + compressLevel = (uint8)compressOptions->compressLeve + } else { + symbol = false; + compressLevel = (uint8)-compressOptions->compressLeve + } + + bool success = false; + uint8 chunkSize = ConvertChunkSize(compressOptions->compressChunkSize, &success); + + uint8 preallocChunks = 0; + if (compressOptions->compressPreallocChunks >= BLCKSZ / compressOptions->compressChunkSize) { + ; + } else { + preallocChunks = (uint8)compressOptions->compressPreallocChunks; + } + node->opt = 0; + SET_COMPRESS_OPTION((*node), (int)compressOptions->compressByteConvert, + (int)compressOptions->compressDiffConvert, + preallocChunks, (int)symbol, compressLevel, algorithm, chunkSize); + } +} \ No newline at end of file diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 0ce4400..78e1158 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -89,6 +89,7 @@ #include "utils/resowner_private.h" #include "utils/snapmgr.h" #include "utils/syscache.h" +#include "storage/page_compression.h" #define RELCACHE_INIT_FILEMAGIC 0x573266 /* version ID value */ @@ -1382,6 +1383,15 @@ RelationInitPhysicalAddr(Relation relation) else relation->rd_firstRelfilenodeSubid = InvalidSubTransactionId; } + + /* + * Setup page compression options + */ + relation->rd_node.opt = 0; + if (relation->rd_options && REL_SUPPORT_COMPRESSED(RELATION)) { + SetupPageCompressForRelation(&relation->rd_node, &((StdRdOptions*)(void*)(relation->rd_options))->compress, + RelationGetRelationName(relation)); + } } /* @@ -3487,7 +3497,8 @@ RelationBuildLocalRelation(const char *relname, bool shared_relation, bool mapped_relation, char relpersistence, - char relkind) + char relkind, + Datum reloptions) { Relation rel; MemoryContext oldcxt; @@ -3666,6 +3677,11 @@ RelationBuildLocalRelation(const char *relname, rel->rd_rel->relam = accessmtd; + /* + * Compress option was set by RelationInitPhysicalAddr() + */ + // + /* * RelationInitTableAccessMethod will do syscache lookups, so we mustn't * run it in CacheMemoryContext. Fortunately, the remaining steps don't diff --git a/src/include/catalog/heap.h b/src/include/catalog/heap.h index 07c5b88..c17e2eb 100644 --- a/src/include/catalog/heap.h +++ b/src/include/catalog/heap.h @@ -60,7 +60,8 @@ extern Relation heap_create(const char *relname, bool allow_system_table_mods, TransactionId *relfrozenxid, MultiXactId *relminmxid, - bool create_storage); + bool create_storage, + Datum reloptions); extern Oid heap_create_with_catalog(const char *relname, Oid relnamespace, diff --git a/src/include/storage/page_compression.h b/src/include/storage/page_compression.h new file mode 100644 index 0000000..5ab4bb7 --- /dev/null +++ b/src/include/storage/page_compression.h @@ -0,0 +1,79 @@ +#ifndef PAGE_COMPRESSION_H +#define PAGE_COMPRESSION_H + +#include "c.h" + +#define KPHC_CMP_BYTE_CONVERT_LEN 1 +#define KPHC_CMP_DIFF_CONVERT_LEN 1 +#define KPHC_CMP_PRE_CHUNK_LEN 3 +#define KPHC_CMP_LEVEL_SYMBOL_LEN 1 +#define KPHC_CMP_LEVEL_LEN 5 +#define KPHC_CMP_ALGORITHM_LEN 3 +#define KPHC_CMP_CHUNK_SIZE_LEN 2 + +#define KPHC_CMP_BYTE_CONVERT_INDEX 0 +#define KPHC_CMP_DIFF_CONVERT_INDEX 1 +#define KPHC_CMP_PRE_CHUNK_INDEX 2 +#define KPHC_CMP_COMPERSS_LEVEL_SYMBOL_INDEX 3 +#define KPHC_CMP_LEVEL_INDEX 4 +#define KPHC_CMP_ALGORITHM_INDEX 5 +#define KPHC_CMP_CHUNK_SIZE_INDEX 6 + +typedef struct CmpBitStuct { + const unsigned int bitLen; + const unsigned int mask; + const unsigned int moveBit; +}KPHCCmpBitStuct; + +const KPHCCmpBitStuct gCmpBitStruct[] = {{KPHC_CMP_BYTE_CONVERT_LEN, 0x01, 15}, + {KPHC_CMP_DIFF_CONVERT_LEN, 0x01, 14}, + {KPHC_CMP_PRE_CHUNK_LEN, 0x07, 11}, + {KPHC_CMP_LEVEL_SYMBOL_LEN, 0x01, 10}, + {KPHC_CMP_LEVEL_LEN, 0x1F, 5}, + {KPHC_CMP_ALGORITHM_LEN, 0x07, 2}, + {KPHC_CMP_CHUNK_SIZE_LEN, 0x03, 0}}; + +typedef struct relFileCompressOption { + unsigned byteConvert : KPHC_CMP_BYTE_CONVERT_LEN; + unsigned diffConvert : KPHC_CMP_DIFF_CONVERT_LEN; + unsigned compressPreallocChunks : KPHC_CMP_PRE_CHUNK_LEN; + unsigned compressLevelSymbol : KPHC_CMP_LEVEL_SYMBOL_LEN; + unsigned compressLevel : KPHC_CMP_LEVEL_LEN; + unsigned compressAlgorithm : KPHC_CMP_ALGORITHM_LEN; + unsigned compressChunkSize : KPHC_CMP_CHUNK_SIZE_LEN; +}KPHCRelFileCompressOption; + +#define SET_COMPRESS_OPTION(node, byteConvert, diffConvert, preChunks, symbol, level ,algorithm, chunkSize) \ + do { \ + (node).opt = 0; \ + (node).opt = (node).opt << gCmpBitStruct[KPHC_CMP_BYTE_CONVERT_INDEX].bitLen; \ + (node).opt = (byteConvert)&gCmpBitStruct[KPHC_CMP_BYTE_CONVERT_INDEX].mask; \ + (node).opt = (node).opt << gCmpBitStruct[KPHC_CMP_DIFF_CONVERT_INDEX].bitLen; \ + (node).opt = (diffConvert)&gCmpBitStruct[KPHC_CMP_DIFF_CONVERT_INDEX].mask; \ + (node).opt = (node).opt << gCmpBitStruct[KPHC_CMP_PRE_CHUNK_INDEX].bitLen; \ + (node).opt = (preChunks)&gCmpBitStruct[KPHC_CMP_PRE_CHUNK_INDEX].mask; \ + (node).opt = (node).opt << gCmpBitStruct[KPHC_CMP_COMPERSS_LEVEL_SYMBOL_INDEX].bitLen; \ + (node).opt = (symbol)&gCmpBitStruct[KPHC_CMP_COMPERSS_LEVEL_SYMBOL_INDEX].mask; \ + (node).opt = (node).opt << gCmpBitStruct[KPHC_CMP_LEVEL_INDEX].bitLen; \ + (node).opt = (level)&gCmpBitStruct[KPHC_CMP_LEVEL_INDEX].mask; \ + (node).opt = (node).opt << gCmpBitStruct[KPHC_CMP_ALGORITHM_INDEX].bitLen; \ + (node).opt = (algorithm)&gCmpBitStruct[KPHC_CMP_ALGORITHM_INDEX].mask; \ + (node).opt = (node).opt << gCmpBitStruct[KPHC_CMP_CHUNK_SIZE_INDEX].bitLen; \ + (node).opt = (chunkSize)&gCmpBitStruct[KPHC_CMP_CHUNK_SIZE_INDEX].mask; \ + } while (0) + +#define SUPPORT_COMPRESSED(relKind, relam) \ + ((relKind) == RELKIND_RELATION || ((relKind) == RELKIND_INDEX && (relam) == BTREE_AM_OID)) + +#define REL_SUPPORT_COMPRESSED(relation) SUPPORT_COMPRESSED((relation)->rd_rel->relkind, (relation)->rd_rel->relam) + +const uint32 INDEX_OF_HALF_BLCKSZ = 0; +const uint32 INDEX_OF_QUARTER_BLCKSZ = 1; +const uint32 INDEX_OF_EIGHTH_BLCKSZ = 2; +const uint32 INDEX_OF_SIXTEENTHS_BLCKSZ = 3; + +uint8 ConvertChunkSize(uint32 compressedChunkSize, bool *success); + +void SetupPageCompressForRelation(RelFileNode *node, PageCpmpressOpts *compress_options, const char *relationName); + +#endif // PAGE_COMPRESSION_H \ No newline at end of file diff --git a/src/include/storage/relfilenode.h b/src/include/storage/relfilenode.h index 4fdc606..1a66dde 100644 --- a/src/include/storage/relfilenode.h +++ b/src/include/storage/relfilenode.h @@ -59,6 +59,7 @@ typedef struct RelFileNode Oid spcNode; /* tablespace */ Oid dbNode; /* database */ Oid relNode; /* relation */ + uint16 opt; /* 2 bytes for compress options */ } RelFileNode; /* diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index a1bc071..05c03d8 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -308,6 +308,23 @@ typedef struct AutoVacOpts float8 analyze_scale_factor; } AutoVacOpts; +/* page compress related reloptions */ +typedef struct PageCompressOpts { + int compressType; + int compressLevel; + uint32 compressChunkSize; + uint32 compressPreallocChunks; + bool compressByteConvert; + bool compressDiffConvert; +} PageCompressOpts; + +typedef enum CompressTypeOption { + COMPRESS_TYPE_NONE = 0, + COMPRESS_TYPE_PGLZ = 1, + COMPRESS_TYPE_ZSTD = 2, + COMPRESS_TYPE_PGZSTD = 3, +} CompressTypeOption; + /* StdRdOptions->vacuum_index_cleanup values */ typedef enum StdRdOptIndexCleanup { @@ -326,6 +343,7 @@ typedef struct StdRdOptions int parallel_workers; /* max number of parallel workers */ StdRdOptIndexCleanup vacuum_index_cleanup; /* controls index vacuuming */ bool vacuum_truncate; /* enables vacuum to truncate a relation */ + PageCompressOpts compress; /* page compress */ } StdRdOptions; #define HEAP_MIN_FILLFACTOR 10 diff --git a/src/include/utils/relcache.h b/src/include/utils/relcache.h index c93d865..82b1026 100644 --- a/src/include/utils/relcache.h +++ b/src/include/utils/relcache.h @@ -108,7 +108,8 @@ extern Relation RelationBuildLocalRelation(const char *relname, bool shared_relation, bool mapped_relation, char relpersistence, - char relkind); + char relkind, + Datum reloptions); /* * Routines to manage assignment of new relfilenode to a relation -- Gitee