From 952a00c2c4dfbf0d0218a4c297f91763d1e05d20 Mon Sep 17 00:00:00 2001 From: MartinChoo <214582617@qq.com> Date: Mon, 30 Jun 2025 20:24:07 +0800 Subject: [PATCH] Support enable page compress Signed-off-by: MartinChoo <214582617@qq.com> --- BUILD.gn | 42 + bundle.json | 6 +- include/sqlite3.h | 1 + include/sqlite3sym.h | 4 +- patch/0001-History-features-on-OH.patch | 13 +- patch/0002-Enable-and-optimize-ICU.patch | 13 +- patch/0003-Busy-debug-and-log-dump.patch | 13 +- patch/0004-Support-meta-recovery.patch | 13 +- ...ity-report-corruption-and-check-page.patch | 13 +- patch/0006-Support-Binlog.patch | 12 +- patch/0007-DatabaseDebugTool.patch | 12 + patch/0008-Rekey-multi-process.patch | 32 +- ...Allow-enable-checksum-through-PRAGMA.patch | 53 +- ...ExtensionLoading-LoadCustomTokenizer.patch | 22 +- patch/0011-Support-compress-db.patch | 1953 +++++++++++++++++ ...h => 0012-Bugfix-on-current-version.patch} | 57 +- patch/BUILD.gn | 1 + unittest/BUILD.gn | 2 + unittest/common.cpp | 91 + unittest/common.h | 32 + unittest/sqlite_compress_test.cpp | 233 ++ unittest/sqlite_test.cpp | 15 +- 22 files changed, 2510 insertions(+), 123 deletions(-) create mode 100644 patch/0011-Support-compress-db.patch rename patch/{0011-Bug-fixes-on-current-version.patch => 0012-Bugfix-on-current-version.patch} (89%) create mode 100644 unittest/common.cpp create mode 100644 unittest/common.h create mode 100644 unittest/sqlite_compress_test.cpp diff --git a/BUILD.gn b/BUILD.gn index 40f18e3..0ba59f3 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -33,6 +33,7 @@ group("libsqlite") { public_deps = [ ":sqlite", ":sqliteicu", + ":sqlitecompressvfs", ] } @@ -81,6 +82,46 @@ ohos_shared_library("sqliteicu") { ] } +ohos_shared_library("sqlitecompressvfs") { + branch_protector_ret = "pac_ret" + sources = [ "$sqlite_patched_dir/src/compressvfs.c" ] + + defines = [ + "NDEBUG=1", + "SQLITE_EXPORT_SYMBOLS", + "SQLITE_ENABLE_PAGE_COMPRESS", + "HARMONY_OS", + ] + cflags_c = [ + "-fvisibility=hidden", + "-Wno-implicit-fallthrough", + ] + if (target_os != "ios") { + ldflags = [ "-Wl,--exclude-libs,ALL" ] + } + deps = [ + ":sqlite", + "//third_party/sqlite/patch:apply_patch", + ] + public_configs = [ ":sqlite_config" ] + configs = [ ":sqlite3_private_config" ] + innerapi_tags = [ "platformsdk_indirect" ] + part_name = "sqlite" + subsystem_name = "thirdparty" + install_images = [ system_base_dir ] + relative_install_dir = "platformsdk" + external_deps = [] + if (is_cross_platform_build) { + if (target_os == "ios") { + deps += [ "//third_party/bounds_checking_function:libsec_shared" ] + } else { + deps += [ "//commonlibrary/c_utils/base:utils" ] + } + } else { + external_deps += [ "c_utils:utils" ] + } +} + ohos_shared_library("sqlite") { branch_protector_ret = "pac_ret" sources = [ @@ -127,6 +168,7 @@ ohos_shared_library("sqlite") { "SQLITE_META_DWR", "SQLITE_ENABLE_BINLOG", "SQLITE_CKSUMVFS_STATIC", + "SQLITE_ENABLE_PAGE_COMPRESS", ] if (sqlite_support_check_pages) { } diff --git a/bundle.json b/bundle.json index 460a9a5..0b9360a 100644 --- a/bundle.json +++ b/bundle.json @@ -36,7 +36,8 @@ "sub_component": [ "//third_party/sqlite:sqlite", "//third_party/sqlite:sqlite3", - "//third_party/sqlite:sqliteicu" + "//third_party/sqlite:sqliteicu", + "//third_party/sqlite:sqlitecompressvfs" ], "inner_kits": [ { @@ -54,6 +55,9 @@ }, { "name": "//third_party/sqlite:sqlite3" + }, + { + "name": "//third_party/sqlite:sqlitecompressvfs" } ], "test": [ diff --git a/include/sqlite3.h b/include/sqlite3.h index 372d949..c12be40 100644 --- a/include/sqlite3.h +++ b/include/sqlite3.h @@ -566,6 +566,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) #define SQLITE_NOTICE_RBU (SQLITE_NOTICE | (3<<8)) #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) +#define SQLITE_WARNING_NOTCOMPRESSDB (SQLITE_WARNING | (3<<8)) #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) #define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) #define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal use only */ diff --git a/include/sqlite3sym.h b/include/sqlite3sym.h index 59893ff..2eb3bbc 100644 --- a/include/sqlite3sym.h +++ b/include/sqlite3sym.h @@ -36,7 +36,7 @@ typedef enum BinlogFileCleanMode { typedef enum { ROW = 0, } Sqlite3BinlogMode; - + typedef struct Sqlite3BinlogConfig { Sqlite3BinlogMode mode; unsigned short fullCallbackThreshold; @@ -59,6 +59,7 @@ struct sqlite3_api_routines_hw { int (*is_support_binlog)(void); int (*replay_binlog)(sqlite3*, sqlite3*); int (*clean_binlog)(sqlite3*, BinlogFileCleanModeE); + int (*compressdb_backup)(sqlite3*, const char*); }; extern const struct sqlite3_api_routines_hw *sqlite3_export_hw_symbols; @@ -71,6 +72,7 @@ extern const struct sqlite3_api_routines_hw *sqlite3_export_hw_symbols; #define sqlite3_is_support_binlog sqlite3_export_hw_symbols->is_support_binlog #define sqlite3_replay_binlog sqlite3_export_hw_symbols->replay_binlog #define sqlite3_clean_binlog sqlite3_export_hw_symbols->clean_binlog +#define sqlite3_compressdb_backup sqlite3_export_hw_symbols->compressdb_backup struct sqlite3_api_routines_cksumvfs { int (*register_cksumvfs)(const char *); diff --git a/patch/0001-History-features-on-OH.patch b/patch/0001-History-features-on-OH.patch index d33eeee..b6a9a13 100644 --- a/patch/0001-History-features-on-OH.patch +++ b/patch/0001-History-features-on-OH.patch @@ -1,15 +1,14 @@ -From f71a3aaddebdcba87ad8af804d680397ecc5fa48 Mon Sep 17 00:00:00 2001 -From: wanghaishuo -Date: Mon, 12 May 2025 15:29:53 +0800 -Subject: [PATCH 1/6] History features on OH +From 1355da4b32bfb2e263e03e953d052086cf40ed20 Mon Sep 17 00:00:00 2001 +From: MartinChoo <214582617@qq.com> +Date: Thu, 3 Jul 2025 22:53:07 +0800 +Subject: [PATCH 01/12] History features on OH -Signed-off-by: wanghaishuo --- src/sqlite3.c | 2451 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 2438 insertions(+), 13 deletions(-) diff --git a/src/sqlite3.c b/src/sqlite3.c -index 7fb290f..ad78006 100644 +index 730b247..b132937 100644 --- a/src/sqlite3.c +++ b/src/sqlite3.c @@ -2915,6 +2915,11 @@ SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); @@ -3033,5 +3032,5 @@ index 7fb290f..ad78006 100644 +/************** End hw export the symbols *****************************************/ +#endif /* SQLITE_EXPORT_SYMBOLS */ -- -2.34.1 +2.47.0.windows.2 diff --git a/patch/0002-Enable-and-optimize-ICU.patch b/patch/0002-Enable-and-optimize-ICU.patch index 15c1115..559c32f 100644 --- a/patch/0002-Enable-and-optimize-ICU.patch +++ b/patch/0002-Enable-and-optimize-ICU.patch @@ -1,9 +1,8 @@ -From 57b5e5dc8b3a7f622e9666c20ec2a7f78997bf3c Mon Sep 17 00:00:00 2001 -From: wanghaishuo -Date: Mon, 12 May 2025 15:24:45 +0800 -Subject: [PATCH 2/6] Enable and optimize ICU +From 733de3813ffb596919bcc72fb8f61422cbacf295 Mon Sep 17 00:00:00 2001 +From: MartinChoo <214582617@qq.com> +Date: Thu, 3 Jul 2025 22:53:41 +0800 +Subject: [PATCH 02/12] Enable and optimize ICU -Signed-off-by: wanghaishuo --- src/sqlite3.c | 1013 ++++------------------------------------------ src/sqlite3icu.c | 888 ++++++++++++++++++++++++++++++++++++++++ @@ -11,7 +10,7 @@ Signed-off-by: wanghaishuo create mode 100644 src/sqlite3icu.c diff --git a/src/sqlite3.c b/src/sqlite3.c -index ad78006..8d8b8ae 100644 +index b132937..efc4cd4 100644 --- a/src/sqlite3.c +++ b/src/sqlite3.c @@ -2502,6 +2502,7 @@ struct sqlite3_mem_methods { @@ -2020,5 +2019,5 @@ index 0000000..b5944d5 +/************** End of fts3_icu.c ********************************************/ \ No newline at end of file -- -2.34.1 +2.47.0.windows.2 diff --git a/patch/0003-Busy-debug-and-log-dump.patch b/patch/0003-Busy-debug-and-log-dump.patch index 15d6408..2dfa3c7 100644 --- a/patch/0003-Busy-debug-and-log-dump.patch +++ b/patch/0003-Busy-debug-and-log-dump.patch @@ -1,15 +1,14 @@ -From c56ae11e3fe85ab6fa71c9d49ac37a433b01fb4b Mon Sep 17 00:00:00 2001 -From: wanghaishuo -Date: Mon, 12 May 2025 15:31:54 +0800 -Subject: [PATCH 3/6] Busy debug and log dump +From 65e08929709648a1ee5d6b0d3b6383634f8669e3 Mon Sep 17 00:00:00 2001 +From: MartinChoo <214582617@qq.com> +Date: Thu, 3 Jul 2025 22:54:18 +0800 +Subject: [PATCH 03/12] Busy debug and log dump -Signed-off-by: wanghaishuo --- src/sqlite3.c | 745 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 731 insertions(+), 14 deletions(-) diff --git a/src/sqlite3.c b/src/sqlite3.c -index 8d8b8ae..574a76a 100644 +index efc4cd4..6f423a9 100644 --- a/src/sqlite3.c +++ b/src/sqlite3.c @@ -879,6 +879,7 @@ SQLITE_API int sqlite3_exec( @@ -1414,5 +1413,5 @@ index 8d8b8ae..574a76a 100644 #ifdef SQLITE_EXPORT_SYMBOLS struct sqlite3_api_routines_hw { -- -2.34.1 +2.47.0.windows.2 diff --git a/patch/0004-Support-meta-recovery.patch b/patch/0004-Support-meta-recovery.patch index d9f6e35..6f6427f 100644 --- a/patch/0004-Support-meta-recovery.patch +++ b/patch/0004-Support-meta-recovery.patch @@ -1,15 +1,14 @@ -From cdf4fb30449a76dc055c92efef0eee7e128ccd56 Mon Sep 17 00:00:00 2001 -From: wanghaishuo -Date: Mon, 12 May 2025 15:33:13 +0800 -Subject: [PATCH 4/6] Support meta recovery +From 45e883bed32094756ff8878964d911cc0cfe9fd3 Mon Sep 17 00:00:00 2001 +From: MartinChoo <214582617@qq.com> +Date: Thu, 3 Jul 2025 22:54:55 +0800 +Subject: [PATCH 04/12] Support meta recovery -Signed-off-by: wanghaishuo --- src/sqlite3.c | 1156 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 1151 insertions(+), 5 deletions(-) diff --git a/src/sqlite3.c b/src/sqlite3.c -index 574a76a..96ed35b 100644 +index 6f423a9..dc09c3c 100644 --- a/src/sqlite3.c +++ b/src/sqlite3.c @@ -786,6 +786,7 @@ SQLITE_API int sqlite3_exec( @@ -1290,5 +1289,5 @@ index 574a76a..96ed35b 100644 #include #include -- -2.34.1 +2.47.0.windows.2 diff --git a/patch/0005-Enhance-dfx-ability-report-corruption-and-check-page.patch b/patch/0005-Enhance-dfx-ability-report-corruption-and-check-page.patch index bc8e8da..62075d3 100644 --- a/patch/0005-Enhance-dfx-ability-report-corruption-and-check-page.patch +++ b/patch/0005-Enhance-dfx-ability-report-corruption-and-check-page.patch @@ -1,9 +1,8 @@ -From ee3361c659ae4f46615bdb9f703cb3469c595214 Mon Sep 17 00:00:00 2001 -From: wanghaishuo -Date: Mon, 12 May 2025 15:42:10 +0800 -Subject: [PATCH 5/6] Enhance dfx ability, report corruption and check pages +From 530b32340de30cfde81712585051e5b8b77ba095 Mon Sep 17 00:00:00 2001 +From: MartinChoo <214582617@qq.com> +Date: Thu, 3 Jul 2025 22:55:22 +0800 +Subject: [PATCH 05/12] Enhance dfx ability, report corruption and check pages -Signed-off-by: wanghaishuo --- ext/misc/cksumvfs.c | 916 ++++++++++++++++++++++++++++++++++++++++++++ src/sqlite3.c | 818 ++++++++++++++++++++++++++++++++++----- @@ -933,7 +932,7 @@ index 0000000..6f4c55c +EXPORT_SYMBOLS const sqlite3_api_routines_cksumvfs *sqlite3_export_cksumvfs_symbols = &sqlite3CksumvfsApis; +#endif diff --git a/src/sqlite3.c b/src/sqlite3.c -index 96ed35b..2a4e87f 100644 +index dc09c3c..e7e8b3c 100644 --- a/src/sqlite3.c +++ b/src/sqlite3.c @@ -2472,6 +2472,21 @@ struct sqlite3_mem_methods { @@ -2410,5 +2409,5 @@ index 96ed35b..2a4e87f 100644 int (*initialize)(); int (*config)(int,...); -- -2.34.1 +2.47.0.windows.2 diff --git a/patch/0006-Support-Binlog.patch b/patch/0006-Support-Binlog.patch index 91fd9bc..35756b4 100644 --- a/patch/0006-Support-Binlog.patch +++ b/patch/0006-Support-Binlog.patch @@ -1,14 +1,14 @@ -From 5604e33fbc2e70d6b636d4a497e40791b8fcc1bd Mon Sep 17 00:00:00 2001 -From: Liu Hongyang -Date: Sat, 5 Jul 2025 16:18:35 +0800 -Subject: [PATCH] Support-Binlog +From 43816530bf2526e61ba871ab7d82ddb524f3bb94 Mon Sep 17 00:00:00 2001 +From: MartinChoo <214582617@qq.com> +Date: Thu, 3 Jul 2025 22:55:59 +0800 +Subject: [PATCH 06/12] Support-Binlog --- src/sqlite3.c | 1510 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 1501 insertions(+), 9 deletions(-) diff --git a/src/sqlite3.c b/src/sqlite3.c -index e7e8b3c..c095863 100644 +index e7e8b3c..311d910 100644 --- a/src/sqlite3.c +++ b/src/sqlite3.c @@ -2938,7 +2938,9 @@ SQLITE_API sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); @@ -1793,5 +1793,5 @@ index e7e8b3c..c095863 100644 EXPORT_SYMBOLS const sqlite3_api_routines *sqlite3_export_symbols = &sqlite3Apis; -- -2.25.1 +2.47.0.windows.2 diff --git a/patch/0007-DatabaseDebugTool.patch b/patch/0007-DatabaseDebugTool.patch index daddcc8..9fc0387 100644 --- a/patch/0007-DatabaseDebugTool.patch +++ b/patch/0007-DatabaseDebugTool.patch @@ -1,3 +1,12 @@ +From ea38cd58b2a3704a2ce6e7e6840cdaa72c24b566 Mon Sep 17 00:00:00 2001 +From: MartinChoo <214582617@qq.com> +Date: Thu, 3 Jul 2025 22:56:30 +0800 +Subject: [PATCH 07/12] DatabaseDebugTool + +--- + src/shell.c | 4 ++++ + 1 file changed, 4 insertions(+) + diff --git a/src/shell.c b/src/shell.c index 7fb190e..c1948b1 100644 --- a/src/shell.c @@ -20,3 +29,6 @@ index 7fb190e..c1948b1 100644 /* Do an initial pass through the command-line argument to locate ** the name of the database file, the name of the initialization file, ** the size of the alternative malloc heap, options affecting commands +-- +2.47.0.windows.2 + diff --git a/patch/0008-Rekey-multi-process.patch b/patch/0008-Rekey-multi-process.patch index 2a6c342..2eee90b 100644 --- a/patch/0008-Rekey-multi-process.patch +++ b/patch/0008-Rekey-multi-process.patch @@ -1,17 +1,17 @@ -From fa5b9fefdd1d5b13ad1953690ed93f21f26be9ac Mon Sep 17 00:00:00 2001 -From: ryne3366 -Date: Fri, 6 Jun 2025 19:42:36 +0800 -Subject: [PATCH 1/2] rekey multi process +From c21f5d9abd672d70f46e22ffb6bb3ef48e7f1c6e Mon Sep 17 00:00:00 2001 +From: MartinChoo <214582617@qq.com> +Date: Thu, 3 Jul 2025 22:57:18 +0800 +Subject: [PATCH 08/12] Rekey multi process --- src/sqlite3.c | 93 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 3 deletions(-) diff --git a/src/sqlite3.c b/src/sqlite3.c -index 6530284..b808dc6 100644 +index 37501f9..425d3d7 100644 --- a/src/sqlite3.c +++ b/src/sqlite3.c -@@ -15692,6 +15692,7 @@ typedef int VList; +@@ -15689,6 +15689,7 @@ typedef int VList; #define RESERVED_BYTE (PENDING_BYTE+1) #define SHARED_FIRST (PENDING_BYTE+2) #define SHARED_SIZE 510 @@ -19,7 +19,7 @@ index 6530284..b808dc6 100644 /* ** Wrapper around OS specific sqlite3_os_init() function. -@@ -38678,6 +38679,10 @@ struct unixFile { +@@ -38682,6 +38683,10 @@ struct unixFile { #endif }; @@ -30,7 +30,7 @@ index 6530284..b808dc6 100644 /* This variable holds the process id (pid) from when the xRandomness() ** method was called. If xOpen() is called from a different process id, ** indicating that a fork() has occurred, the PRNG will be reset. -@@ -39623,6 +39628,9 @@ struct unixInodeInfo { +@@ -39627,6 +39632,9 @@ struct unixInodeInfo { sqlite3_mutex *pLockMutex; /* Hold this mutex for... */ int nShared; /* Number of SHARED locks held */ int nLock; /* Number of outstanding file locks */ @@ -40,7 +40,7 @@ index 6530284..b808dc6 100644 unsigned char eFileLock; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ unsigned char bProcessLock; /* An exclusive process lock is held */ UnixUnusedFd *pUnused; /* Unused file descriptors to close */ -@@ -61693,6 +61701,9 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ +@@ -61697,6 +61705,9 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ PAGERTRACE(("CLOSE %d\n", PAGERID(pPager))); IOTRACE(("CLOSE %p\n", pPager)) sqlite3OsClose(pPager->jfd); @@ -50,7 +50,7 @@ index 6530284..b808dc6 100644 sqlite3OsClose(pPager->fd); sqlite3PageFree(pTmp); sqlite3PcacheClose(pPager->pPCache); -@@ -255484,6 +255495,54 @@ void sqlite3CodecDetach(void *ctx){ +@@ -255500,6 +255511,54 @@ void sqlite3CodecDetach(void *ctx){ return; } @@ -105,7 +105,7 @@ index 6530284..b808dc6 100644 int sqlite3CodecAttach(sqlite3* db, int nDb, const void *pKey, int nKey){ if(db == NULL){ return SQLITE_ERROR; -@@ -255503,6 +255562,18 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *pKey, int nKey){ +@@ -255519,6 +255578,18 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *pKey, int nKey){ return SQLITE_ERROR; } sqlite3_mutex_enter(db->mutex); @@ -124,7 +124,7 @@ index 6530284..b808dc6 100644 #ifdef SQLITE_CODEC_ATTACH_CHANGED int rc = sqlite3CodecInitContext(ctx, p, pKey, nKey, nDb); #else -@@ -255510,6 +255581,7 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *pKey, int nKey){ +@@ -255526,6 +255597,7 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *pKey, int nKey){ #endif /* SQLITE_CODEC_ATTACH_CHANGED */ if(rc != SQLITE_OK){ sqlite3_free(ctx); @@ -132,7 +132,7 @@ index 6530284..b808dc6 100644 return rc; } sqlite3PagerSetCodec(sqlite3BtreePager(p), sqlite3Codec, NULL, sqlite3CodecDetach, (void *)ctx); -@@ -255518,7 +255590,7 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *pKey, int nKey){ +@@ -255534,7 +255606,7 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *pKey, int nKey){ p->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; sqlite3BtreeSetPageSize(p, ctx->readCtx->codecConst.cipherPageSize, ctx->readCtx->codecConst.reserveSize, 0); sqlite3BtreeSecureDelete(p, 1); @@ -141,7 +141,7 @@ index 6530284..b808dc6 100644 sqlite3BtreeSetAutoVacuum(p, SQLITE_DEFAULT_AUTOVACUUM); } -@@ -255589,7 +255661,20 @@ int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey){ +@@ -255605,7 +255677,20 @@ int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey){ return rc; } sqlite3_mutex_enter(db->mutex); @@ -163,7 +163,7 @@ index 6530284..b808dc6 100644 sqlite3PagerPagecount(pPager, &pageCount); // support hmac algo changed by using rekey operation int oldHmacAlgo = ctx->writeCtx->codecConst.hmacAlgo; -@@ -255624,8 +255709,10 @@ int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey){ +@@ -255640,8 +255725,10 @@ int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey){ } (void)sqlite3BtreeRollback(p, SQLITE_ABORT_ROLLBACK, 0); } @@ -176,5 +176,5 @@ index 6530284..b808dc6 100644 } -- -2.34.1 +2.47.0.windows.2 diff --git a/patch/0009-Allow-enable-checksum-through-PRAGMA.patch b/patch/0009-Allow-enable-checksum-through-PRAGMA.patch index fc422b9..bb63f29 100644 --- a/patch/0009-Allow-enable-checksum-through-PRAGMA.patch +++ b/patch/0009-Allow-enable-checksum-through-PRAGMA.patch @@ -1,9 +1,8 @@ -From 6795a730af198fed664ac8418b36368048c4aa4b Mon Sep 17 00:00:00 2001 +From 1136400a2cdcdbd784cb459cfc6f123f14f3cc58 Mon Sep 17 00:00:00 2001 From: MartinChoo <214582617@qq.com> -Date: Sat, 7 Jun 2025 22:25:48 +0800 -Subject: [PATCH] Allow enable checksum through PRAGMA +Date: Thu, 3 Jul 2025 22:57:51 +0800 +Subject: [PATCH 09/12] Allow enable checksum through PRAGMA -Signed-off-by: MartinChoo <214582617@qq.com> --- ext/misc/cksumvfs.c | 185 ++++++++++++++++++++++++++++++++++++-------- src/sqlite3.c | 175 ++++++++++++++++++++++++++++++++--------- @@ -377,10 +376,10 @@ index 6f4c55c..a23a997 100644 int (*register_cksumvfs)(const char *); int (*unregister_cksumvfs)(); diff --git a/src/sqlite3.c b/src/sqlite3.c -index 3be1113..80d7979 100644 +index 425d3d7..24e3247 100644 --- a/src/sqlite3.c +++ b/src/sqlite3.c -@@ -139859,6 +139859,10 @@ static int integrityCheckResultRow(Vdbe *v){ +@@ -139875,6 +139875,10 @@ static int integrityCheckResultRow(Vdbe *v){ return addr; } @@ -391,7 +390,7 @@ index 3be1113..80d7979 100644 /* ** Process a pragma statement. ** -@@ -139981,7 +139985,15 @@ SQLITE_PRIVATE void sqlite3Pragma( +@@ -139997,7 +140001,15 @@ SQLITE_PRIVATE void sqlite3Pragma( /* PragmaMetaDoubleWrie executes internal */ goto pragma_out; } @@ -408,7 +407,7 @@ index 3be1113..80d7979 100644 /* Locate the pragma in the lookup table */ pPragma = pragmaLocate(zLeft); if( pPragma==0 ){ -@@ -184243,16 +184255,12 @@ SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, vo +@@ -184259,16 +184271,12 @@ SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, vo *(unsigned int*)pArg = sqlite3PagerDataVersion(pPager); rc = SQLITE_OK; }else if( op==SQLITE_FCNTL_RESERVE_BYTES ){ @@ -425,7 +424,7 @@ index 3be1113..80d7979 100644 }else if( op==SQLITE_FCNTL_RESET_CACHE ){ sqlite3BtreeClearCache(pBtree); rc = SQLITE_OK; -@@ -256078,7 +256086,71 @@ export_finish: +@@ -256094,7 +256102,71 @@ export_finish: return; } /************** End file hw_codec.c *****************************************/ @@ -498,7 +497,7 @@ index 3be1113..80d7979 100644 #ifdef SQLITE_META_DWR #define META_DWR_MAX_PAGES 500 #define META_DWR_MAGIC 0x234A86D9 -@@ -256110,7 +256182,7 @@ typedef struct MetaDwrHdr { +@@ -256126,7 +256198,7 @@ typedef struct MetaDwrHdr { Pgno *pages; u32 pageBufSize; u8 hdrValid; @@ -507,7 +506,7 @@ index 3be1113..80d7979 100644 u16 needSync; i64 lastSyncTime; } MetaDwrHdr; -@@ -256124,7 +256196,7 @@ typedef struct MetaDwrHdr { +@@ -256140,7 +256212,7 @@ typedef struct MetaDwrHdr { static int MetaDwrHeaderSimpleCheck(Pager *pPager, MetaDwrHdr *hdr) { #if SQLITE_OS_UNIX if (hdr->checkFileId) { @@ -516,7 +515,7 @@ index 3be1113..80d7979 100644 if (fd == NULL || fd->pInode == NULL || pPager->pVfs == NULL) { return SQLITE_INTERNAL; } -@@ -256134,7 +256206,7 @@ static int MetaDwrHeaderSimpleCheck(Pager *pPager, MetaDwrHdr *hdr) { +@@ -256150,7 +256222,7 @@ static int MetaDwrHeaderSimpleCheck(Pager *pPager, MetaDwrHdr *hdr) { return SQLITE_IOERR_DATA; } } @@ -525,7 +524,7 @@ index 3be1113..80d7979 100644 if (hdr->pageCnt > META_DWR_MAX_PAGES || hdr->version != META_DWR_VERSION || hdr->magic != META_DWR_MAGIC || hdr->checkSum != META_DWR_MAGIC) { sqlite3_log(SQLITE_IOERR_DATA, "Meta dwr file check wrong pageCnt %u, version %u, magic %u, checkSum %u", -@@ -256366,7 +256438,15 @@ static MetaDwrHdr *AllocInitMetaHeaderDwr(Pager *pPager) { +@@ -256382,7 +256454,15 @@ static MetaDwrHdr *AllocInitMetaHeaderDwr(Pager *pPager) { MetaDwrReleaseHdr(hdr); return NULL; } @@ -542,7 +541,7 @@ index 3be1113..80d7979 100644 return hdr; } -@@ -256379,7 +256459,7 @@ static void MetaDwrCloseFile(Pager *pPager) { +@@ -256395,7 +256475,7 @@ static void MetaDwrCloseFile(Pager *pPager) { osMunmap(pPager->metaMapPage, META_DWR_HEADER_PAGE_SIZE); pPager->metaMapPage = NULL; } @@ -551,7 +550,7 @@ index 3be1113..80d7979 100644 if (pPager->metaHdr && pPager->metaHdr->needSync > 0) { (void)sqlite3OsSync(pPager->metaFd, SQLITE_SYNC_NORMAL); } -@@ -256474,10 +256554,10 @@ static void MetaDwrUpdateHeaderDbInfo(BtShared *pBt) { +@@ -256490,10 +256570,10 @@ static void MetaDwrUpdateHeaderDbInfo(BtShared *pBt) { } else { hdr->mxFrameInWal = 0; } @@ -564,7 +563,7 @@ index 3be1113..80d7979 100644 if (fd == NULL || fd->pInode == NULL) { sqlite3_log(SQLITE_WARNING_DUMP, "update meta header invalid fd"); hdr->hdrValid = 0; -@@ -256485,7 +256565,7 @@ static void MetaDwrUpdateHeaderDbInfo(BtShared *pBt) { +@@ -256501,7 +256581,7 @@ static void MetaDwrUpdateHeaderDbInfo(BtShared *pBt) { } hdr->dbFileInode = fd->pInode->fileId.ino; } @@ -573,7 +572,7 @@ index 3be1113..80d7979 100644 hdr->freeListPageNo = sqlite3Get4byte(dbHdrInfo + 4); hdr->freeListPageCnt = sqlite3Get4byte(dbHdrInfo + 8); hdr->schemaCookie = sqlite3Get4byte(dbHdrInfo + 12); -@@ -256581,7 +256661,7 @@ static int MetaDwrWriteOnePage(Btree *pBt, PgHdr *pPage, MetaDwrHdr *hdr, u8 cur +@@ -256597,7 +256677,7 @@ static int MetaDwrWriteOnePage(Btree *pBt, PgHdr *pPage, MetaDwrHdr *hdr, u8 cur return SQLITE_NOMEM; #else pData = pPage->pData; @@ -582,7 +581,7 @@ index 3be1113..80d7979 100644 rc = sqlite3OsWrite(pPager->metaFd, pData, pageSz, ofs); if (rc != SQLITE_OK) { return rc; -@@ -256643,7 +256723,7 @@ static int MetaDwrCheckPerm(sqlite3_vfs *pVfs, u8 openCreate, char *metaPath) { +@@ -256659,7 +256739,7 @@ static int MetaDwrCheckPerm(sqlite3_vfs *pVfs, u8 openCreate, char *metaPath) { if (openCreate) { sqlite3_log(SQLITE_WARNING_DUMP, "Meta double write disabled, sysno %d", errno); } @@ -591,7 +590,7 @@ index 3be1113..80d7979 100644 return rc; } -@@ -256686,7 +256766,7 @@ static int MetaDwrOpenFile(Pager *pPager, u8 openCreate) { +@@ -256702,7 +256782,7 @@ static int MetaDwrOpenFile(Pager *pPager, u8 openCreate) { pPager->metaMapPage = page; } } @@ -600,7 +599,7 @@ index 3be1113..80d7979 100644 pPager->metaFd = metaFd; INIT_META_OUT: sqlite3EndBenignMalloc(); -@@ -256810,7 +256890,7 @@ static int MetaDwrRecoverHeadPage( +@@ -256826,7 +256906,7 @@ static int MetaDwrRecoverHeadPage( goto RELEASE_OUT; } } @@ -609,7 +608,7 @@ index 3be1113..80d7979 100644 rc = SQLITE_NOTADB; for (u32 i = 0; i < hdr->pageCnt; i++) { if (hdr->pages[i] != pgno) { -@@ -257060,15 +257140,24 @@ CHK_RESTORE_OUT: +@@ -257076,15 +257156,24 @@ CHK_RESTORE_OUT: static inline u8 IsConnectionValidForCheck(Pager *pPager) { #if SQLITE_OS_UNIX @@ -642,7 +641,7 @@ index 3be1113..80d7979 100644 #endif } -@@ -257083,7 +257172,7 @@ static int MetaDwrOpenAndCheck(Btree *pBt) +@@ -257099,7 +257188,7 @@ static int MetaDwrOpenAndCheck(Btree *pBt) if (pPager->xCodec) { return SQLITE_OK; } @@ -651,7 +650,7 @@ index 3be1113..80d7979 100644 sqlite3BtreeEnter(pBt); int rc = SQLITE_OK; int openedTransaction = 0; -@@ -257128,7 +257217,7 @@ static void MetaDwrDisable(Btree *pBt) +@@ -257144,7 +257233,7 @@ static void MetaDwrDisable(Btree *pBt) if (pPager->xCodec) { return; } @@ -660,7 +659,7 @@ index 3be1113..80d7979 100644 sqlite3BtreeEnter(pBt); MetaDwrCloseFile(pPager); MetaDwrReleaseHdr(pPager->metaHdr); -@@ -257143,7 +257232,8 @@ static void MetaDwrDisable(Btree *pBt) +@@ -257159,7 +257248,8 @@ static void MetaDwrDisable(Btree *pBt) } sqlite3BtreeLeave(pBt); } @@ -670,7 +669,7 @@ index 3be1113..80d7979 100644 #if SQLITE_OS_UNIX #include #include -@@ -257352,10 +257442,17 @@ static void DumpLocksByWal(Wal *pWal) +@@ -257368,10 +257458,17 @@ static void DumpLocksByWal(Wal *pWal) sqlite3_log(SQLITE_ERROR, "Wal ptr is NULL!"); return; } @@ -690,7 +689,7 @@ index 3be1113..80d7979 100644 } #endif /* #ifndef SQLITE_OMIT_WAL */ -@@ -257365,13 +257462,20 @@ static void DumpLocksByPager(Pager *pPager) +@@ -257381,13 +257478,20 @@ static void DumpLocksByPager(Pager *pPager) sqlite3_log(SQLITE_ERROR, "Pager ptr is NULL!"); return; } @@ -714,7 +713,7 @@ index 3be1113..80d7979 100644 #endif /* #ifndef SQLITE_OMIT_WAL */ } #endif /* SQLITE_OS_UNIX */ -@@ -258307,7 +258411,8 @@ static const sqlite3_api_routines_cksumvfs sqlite3CksumvfsApis = { +@@ -258484,7 +258588,8 @@ static const sqlite3_api_routines_cksumvfs sqlite3CksumvfsApis = { }; EXPORT_SYMBOLS const sqlite3_api_routines_cksumvfs *sqlite3_export_cksumvfs_symbols = &sqlite3CksumvfsApis; diff --git a/patch/0010-EnableExtensionLoading-LoadCustomTokenizer.patch b/patch/0010-EnableExtensionLoading-LoadCustomTokenizer.patch index 571dd7a..8176939 100644 --- a/patch/0010-EnableExtensionLoading-LoadCustomTokenizer.patch +++ b/patch/0010-EnableExtensionLoading-LoadCustomTokenizer.patch @@ -1,5 +1,14 @@ +From 4df2a5e625753f2cb9c211617e91f606fdd6d4c6 Mon Sep 17 00:00:00 2001 +From: MartinChoo <214582617@qq.com> +Date: Thu, 3 Jul 2025 22:58:29 +0800 +Subject: [PATCH 10/12] EnableExtensionLoading-LoadCustomTokenizer + +--- + src/shell.c | 32 ++++++++++++++++++++++++++++++++ + 1 file changed, 32 insertions(+) + diff --git a/src/shell.c b/src/shell.c -index 7fb190e..6dc5087 100644 +index c1948b1..cedf118 100644 --- a/src/shell.c +++ b/src/shell.c @@ -118,6 +118,7 @@ typedef unsigned short int u16; @@ -8,9 +17,9 @@ index 7fb190e..6dc5087 100644 #include +#include #include "sqlite3.h" + #include "sqlite3sym.h" typedef sqlite3_int64 i64; - typedef sqlite3_uint64 u64; -@@ -22189,6 +22190,32 @@ static void open_db(ShellState *p, int openFlags){ +@@ -22190,6 +22191,32 @@ static void open_db(ShellState *p, int openFlags){ } sqlite3_db_config(p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, (int)0, (int*)0); @@ -43,7 +52,7 @@ index 7fb190e..6dc5087 100644 /* Reflect the use or absence of --unsafe-testing invocation. */ { int testmode_on = ShellHasFlag(p,SHFLG_TestingMode); -@@ -28237,6 +28264,8 @@ static int line_is_complete(char *zSql, int nSql){ +@@ -28238,6 +28265,8 @@ static int line_is_complete(char *zSql, int nSql){ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ int rc; char *zErrMsg = 0; @@ -52,7 +61,7 @@ index 7fb190e..6dc5087 100644 open_db(p, 0); if( ShellHasFlag(p,SHFLG_Backslash) ) resolve_backslashes(zSql); -@@ -28244,6 +28273,9 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ +@@ -28245,6 +28274,9 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ BEGIN_TIMER; rc = shell_exec(p, zSql, &zErrMsg); END_TIMER; @@ -62,3 +71,6 @@ index 7fb190e..6dc5087 100644 if( rc || zErrMsg ){ char zPrefix[100]; const char *zErrorTail; +-- +2.47.0.windows.2 + diff --git a/patch/0011-Support-compress-db.patch b/patch/0011-Support-compress-db.patch new file mode 100644 index 0000000..674f7b8 --- /dev/null +++ b/patch/0011-Support-compress-db.patch @@ -0,0 +1,1953 @@ +From 9b6d3d2c1c6a34701067ba0069dd69f180c21de2 Mon Sep 17 00:00:00 2001 +From: MartinChoo <214582617@qq.com> +Date: Thu, 3 Jul 2025 23:12:46 +0800 +Subject: [PATCH 11/12] Support compress db + +--- + ext/misc/cksumvfs.c | 4 +- + src/compressvfs.c | 1100 +++++++++++++++++++++++++++++++++++++++++++ + src/sqlite3.c | 443 ++++++++++++++++- + 3 files changed, 1531 insertions(+), 16 deletions(-) + create mode 100644 src/compressvfs.c + +diff --git a/ext/misc/cksumvfs.c b/ext/misc/cksumvfs.c +index a23a997..1a801fb 100644 +--- a/ext/misc/cksumvfs.c ++++ b/ext/misc/cksumvfs.c +@@ -966,7 +966,7 @@ static int cksmRegisterVfs(void){ + cksm_vfs.iVersion = pOrig->iVersion; + cksm_vfs.pAppData = pOrig; + cksm_vfs.szOsFile = pOrig->szOsFile + sizeof(CksmFile); +- rc = sqlite3_vfs_register(&cksm_vfs, 1); ++ rc = sqlite3_vfs_register(&cksm_vfs, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))cksmRegisterFunc); + } +@@ -1019,7 +1019,7 @@ int sqlite3_cksumvfs_init( + #endif /* !defined(SQLITE_CKSUMVFS_STATIC) */ + + #ifdef SQLITE_CKSUMVFS_STATIC +-sqlite3_file *sqlite3_get_orig_file(sqlite3_file *file) { ++sqlite3_file *cksmvfsGetOrigFile(sqlite3_file *file) { + return ORIGFILE(file); + } + +diff --git a/src/compressvfs.c b/src/compressvfs.c +new file mode 100644 +index 0000000..b1d0ac6 +--- /dev/null ++++ b/src/compressvfs.c +@@ -0,0 +1,1100 @@ ++/************** Begin file compressvfs.c *********************************************/ ++/* ++** 2025 June 26 ++** ++** The author disclaims copyright to this source code. In place of ++** a legal notice, here is a blessing: ++** ++** May you do good and not evil. ++** May you find forgiveness for yourself and forgive others. ++** May you share freely, never taking more than you give. ++** ++************************************************************************* ++** ++** This file implements a VFS shim that compress each page's data ++** of an SQLite database file. It will Open a OutterDB which used to ++** manager compressed page data and create a vfs_pages table into OutterDB. ++** When read pages's data, the data will be selected from vfs_pages and ++** decompress by compression. ++** ++** COMPILING ++** ++** This extension requires SQLite 3.44.4 or later. It uses the ++** sqlite3_database_file_object() interface which was added in ++** version 3.44.4, so it will not link with an earlier version of ++** SQLite. ++** ++** To build this extension as a separately loaded shared library or ++** DLL, use compiler command-lines similar to the following: ++** ++** (linux) gcc -fPIC -shared compressvfs.c -o sqlitecompressvfs.z.so ++** ++** You may want to add additional compiler options, of course, ++** according to the needs of your project. ++** ++** LOADING ++** ++** To use this extension as a shared library, you first have to ++** open SQLite database connection with compressvfs. ++** It will load autometically when connect to OutterDB. ++** And then compressvfs will be reegistered before open db connection. ++** After first used , all subsequent databse connections that are opened ++** will include this extension. ++** ++** Compressvfs is a VFS Shim. When loaded, "compressvfs" will registered based on ++** default VFS. This is normally what you want. ++** ++** USING ++** ++** Open database connections using the sqlite3_open() with uri "?vfs=compressvfs" or ++** sqlite3_open_v2() interfaces with zVfs parameter. ++** ++** For example: ++** ++** sqlite3 *db; ++** sqlite3_open("file:example.db?vfs=compressvfs", &db); ++** or ++** sqlite3_open_v2("example.db", &db, flag, "compressvfs"); ++** ++** After open databse connections with compressvfs, page data will be auto compressed. ++** And data can be read and written normally. ++** ++*/ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++#include ++SQLITE_EXTENSION_INIT1 ++#include ++#include ++#include ++#include ++#include ++#include "securec.h" ++#ifndef _WIN32 ++#include ++#endif ++ ++// hw export the symbols ++#ifdef SQLITE_EXPORT_SYMBOLS ++#if defined(__GNUC__) ++# define EXPORT_SYMBOLS __attribute__ ((visibility ("default"))) ++#elif defined(_MSC_VER) ++# define EXPORT_SYMBOLS __declspec(dllexport) ++#else ++# define EXPORT_SYMBOLS ++#endif ++#endif ++ ++/* ++** Useful datatype ++*/ ++#ifndef UINT32_TYPE ++# ifdef HAVE_UINT32_T ++# define UINT32_TYPE uint32_t ++# else ++# define UINT32_TYPE unsigned int ++# endif ++#endif ++#ifndef UINT16_TYPE ++# ifdef HAVE_UINT16_T ++# define UINT16_TYPE uint16_t ++# else ++# define UINT16_TYPE unsigned short int ++# endif ++#endif ++#ifndef INT16_TYPE ++# ifdef HAVE_INT16_T ++# define INT16_TYPE int16_t ++# else ++# define INT16_TYPE short int ++# endif ++#endif ++#ifndef UINT8_TYPE ++# ifdef HAVE_UINT8_T ++# define UINT8_TYPE uint8_t ++# else ++# define UINT8_TYPE unsigned char ++# endif ++#endif ++#ifndef INT8_TYPE ++# ifdef HAVE_INT8_T ++# define INT8_TYPE int8_t ++# else ++# define INT8_TYPE signed char ++# endif ++#endif ++#ifndef LONGDOUBLE_TYPE ++# define LONGDOUBLE_TYPE long double ++#endif ++#ifdef SQLITE_INT64_TYPE ++ typedef SQLITE_INT64_TYPE sqlite_int64; ++# ifdef SQLITE_UINT64_TYPE ++ typedef SQLITE_UINT64_TYPE sqlite_uint64; ++# else ++ typedef unsigned SQLITE_INT64_TYPE sqlite_uint64; ++# endif ++#elif defined(_MSC_VER) || defined(__BORLANDC__) ++ typedef __int64 sqlite_int64; ++ typedef unsigned __int64 sqlite_uint64; ++#else ++ typedef long long int sqlite_int64; ++ typedef unsigned long long int sqlite_uint64; ++#endif ++typedef sqlite_int64 sqlite3_int64; ++typedef sqlite_uint64 sqlite3_uint64; ++typedef sqlite_int64 i64; /* 8-byte signed integer */ ++typedef sqlite_uint64 u64; /* 8-byte unsigned integer */ ++typedef UINT32_TYPE u32; /* 4-byte unsigned integer */ ++typedef UINT16_TYPE u16; /* 2-byte unsigned integer */ ++typedef INT16_TYPE i16; /* 2-byte signed integer */ ++typedef UINT8_TYPE u8; /* 1-byte unsigned integer */ ++typedef INT8_TYPE i8; /* 1-byte signed integer */ ++ ++typedef u32 Pgno; ++/* VFS's name */ ++#define COMPRESS_VFS_NAME "compressvfs" ++ ++/* COMPRESSION OPTIONS */ ++#define COMPRESSION_UNDEFINED 0 ++#define COMPRESSION_BROTLI 1 ++#define COMPRESSION_ZSTD 2 ++ ++#define COMPRESSION_SQL_MAX_LENGTH 100 ++ ++#define SQLITE_SHMMAP_IS_WRITE 0x00000001 /*Flag for xShmMap, extend file if necessary */ ++#define SQLITE_OPEN_COMPRESS_SHM 0x00010000 /*Flag for xShmMap, need to rename shm file */ ++ ++typedef struct compress_info compress_info; ++struct compress_info{ ++ sqlite3_vfs *pRootVfs; /* Ptr to default VFS */ ++ sqlite3* pDb; /* Ptr to OutterDB */ ++ u8 bOutterDbOpen; /* True to OutterDB is opened */ ++ u8 bSubDbOpen; /* True to SubDB is opened */ ++}; ++ ++/* An open file */ ++typedef struct{ ++ sqlite3_file base; /* IO methods */ ++ compress_info *pInfo; /* Ptr to compress info */ ++ u8 bBegin; /* True to xSync() need commit */ ++ u8 compression; /* Compression options */ ++ int page_size; /* Uncompressed page size */ ++} CompressFile; ++ ++ ++static int compressClose(sqlite3_file *pFile); ++static int compressRead(sqlite3_file *pFile, void *pBuf, int iAmt, sqlite_int64 iOfst); ++static int compressWrite(sqlite3_file *pFile, const void *pBuf, int iAmt, sqlite_int64 iOfst); ++static int compressTruncate(sqlite3_file *pFile, sqlite_int64 size); ++static int compressSync(sqlite3_file *pFile, int flags); ++static int compressFileSize(sqlite3_file *pFile, i64 *pSize); ++static int compressLock(sqlite3_file *pFile, int eFileLock); ++static int compressUnlock(sqlite3_file *pFile, int eFileLock); ++static int compressCheckReservedLock(sqlite3_file *pFile, int *pResOut); ++static int compressFileControl(sqlite3_file *pFile, int op, void *pArg); ++static int compressSectorSize(sqlite3_file *pFile); ++static int compressDeviceCharacteristics(sqlite3_file *pFile); ++static int compressShmMap(sqlite3_file *pFile, int iPg, int pgsz, int fileFlag, void volatile **pp); ++static int compressShmLock(sqlite3_file *pFile, int offset, int n, int flags); ++static void compressShmBarrier(sqlite3_file *pFile); ++static int compressShmUnmap(sqlite3_file *pFile, int deleteFlag); ++static int compressFetch(sqlite3_file *pFile, sqlite_int64 iOfst, int iAmt, void **pp); ++static int compressUnfetch(sqlite3_file *pFile, sqlite_int64 iOfst, void *pPage); ++ ++static sqlite3_vfs compress_vfs = {0}; ++static compress_info compress_vfs_info = {0}; ++static sqlite3_io_methods compress_io_methods = { ++ 3, ++ compressClose, ++ compressRead, ++ compressWrite, ++ compressTruncate, ++ compressSync, ++ compressFileSize, ++ compressLock, ++ compressUnlock, ++ compressCheckReservedLock, ++ compressFileControl, ++ compressSectorSize, ++ compressDeviceCharacteristics, ++ compressShmMap, ++ compressShmLock, ++ compressShmBarrier, ++ compressShmUnmap, ++ compressFetch, ++ compressUnfetch ++}; ++ ++/*----------------------------brotli header begin----------------------------*/ ++#define BROTLI_BOOL int ++#define BROTLI_TRUE 1 ++#define BROTLI_FALSE 0 ++#define BROTLI_MAX_WINDOW_BITS 24 ++typedef enum{ ++ // Decoding error ++ BROTLI_DECODER_RESULT_ERROR = 0, ++ // Decoding successfully completed. ++ BROTLI_DECODER_RESULT_SUCCESS = 1, ++ // Should be called again more input. ++ BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT = 2, ++ // Should be called again more output. ++ BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT = 3, ++} BrotliDecoderResult; ++ ++typedef BROTLI_BOOL (*brotliCompress_ptr)(u32, u32, size_t, const u8*, size_t*, u8*); ++typedef BrotliDecoderResult (*brotliDecompress_ptr)(size_t, const u8*, size_t*, u8*); ++/*----------------------------brotli header end----------------------------*/ ++ ++/*----------------------------zstd header begin----------------------------*/ ++typedef size_t (*zstdCompress_ptr)(void*, size_t, const void*, size_t, int); ++typedef size_t (*zstdDecompress_ptr)(void*, size_t, const void*, size_t); ++/*----------------------------zstd header begin----------------------------*/ ++ ++/* ++** Access to a lower-level VFS that (might) implement dynamic loading, access to randomness, etc. ++*/ ++#define ORIGFILE(p) ((sqlite3_file*)(((CompressFile*)(p))+1)) ++ ++static u32 g_compress_algo_load = COMPRESSION_UNDEFINED; ++static void *g_compress_algo_library = NULL; ++typedef size_t (*compressBound_ptr)(size_t); ++static compressBound_ptr compressBoundPtr = NULL; ++ ++static brotliCompress_ptr brotliCompressPtr = NULL; ++static brotliDecompress_ptr brotliDecompressPtr = NULL; ++static zstdCompress_ptr zstdCompressPtr = NULL; ++static zstdDecompress_ptr zstdDecompressPtr = NULL; ++ ++static int loadBrotliExtension(){ ++ g_compress_algo_library = dlopen("libbrotli_shared.z.so", RTLD_LAZY); ++ if( g_compress_algo_library==NULL ){ ++ sqlite3_log(SQLITE_NOTICE, "load brotli so failed :%s", dlerror()); ++ return SQLITE_ERROR; ++ } ++ compressBoundPtr = (compressBound_ptr)dlsym(g_compress_algo_library, "BrotliEncoderMaxCompressedSize"); ++ if( compressBoundPtr==NULL ){ ++ goto failed; ++ } ++ brotliCompressPtr = (brotliCompress_ptr)dlsym(g_compress_algo_library, "BrotliEncoderCompress"); ++ if( brotliCompressPtr==NULL ){ ++ goto failed; ++ } ++ brotliDecompressPtr = (brotliDecompress_ptr)dlsym(g_compress_algo_library, "BrotliDecoderDecompress"); ++ if( brotliDecompressPtr==NULL ){ ++ goto failed; ++ } ++ g_compress_algo_load = COMPRESSION_BROTLI; ++ return SQLITE_OK; ++ ++ failed: ++ sqlite3_log(SQLITE_NOTICE, "load brotli dlsym failed :%s", dlerror()); ++ compressBoundPtr = NULL; ++ brotliCompressPtr = NULL; ++ brotliDecompressPtr = NULL; ++ dlclose(g_compress_algo_library); ++ return SQLITE_ERROR; ++} ++ ++static int loadZstsExtension(){ ++ g_compress_algo_library = dlopen("libzstd_shared.z.so", RTLD_LAZY); ++ if( g_compress_algo_library==NULL ){ ++ sqlite3_log(SQLITE_NOTICE, "load zstd so failed :%s", dlerror()); ++ return SQLITE_ERROR; ++ } ++ compressBoundPtr = (compressBound_ptr)dlsym(g_compress_algo_library, "ZSTD_compressBound"); ++ if( compressBoundPtr==NULL ){ ++ goto failed; ++ } ++ zstdCompressPtr = (zstdCompress_ptr)dlsym(g_compress_algo_library, "ZSTD_compress"); ++ if( zstdCompressPtr==NULL ){ ++ goto failed; ++ } ++ zstdDecompressPtr = (zstdDecompress_ptr)dlsym(g_compress_algo_library, "ZSTD_decompress"); ++ if( zstdDecompressPtr==NULL ){ ++ goto failed; ++ } ++ g_compress_algo_load = COMPRESSION_ZSTD; ++ return SQLITE_OK; ++ ++ failed: ++ sqlite3_log(SQLITE_NOTICE, "load zstd dlsym failed :%s", dlerror()); ++ compressBoundPtr = NULL; ++ zstdCompressPtr = NULL; ++ zstdDecompressPtr = NULL; ++ dlclose(g_compress_algo_library); ++ return SQLITE_ERROR; ++} ++ ++static int loadCompressAlgorithmExtension(u8 compression){ ++ if( g_compress_algo_load!=0u ){ ++ return SQLITE_OK; ++ } ++#ifndef _WIN32 ++ if( compression==COMPRESSION_UNDEFINED ){ ++ if( loadZstsExtension()==SQLITE_ERROR ){ ++ sqlite3_log(SQLITE_NOTICE, "load zstd failed :%s", dlerror()); ++ if( loadBrotliExtension()==SQLITE_ERROR ){ ++ sqlite3_log(SQLITE_ERROR, "load compress so failed :%s", dlerror()); ++ return SQLITE_ERROR; ++ } ++ } ++ }else if( compression==COMPRESSION_BROTLI ){ ++ if( loadBrotliExtension()==SQLITE_ERROR ){ ++ sqlite3_log(SQLITE_ERROR, "load brotli so failed :%s", dlerror()); ++ return SQLITE_ERROR; ++ } ++ }else if( compression==COMPRESSION_ZSTD ){ ++ if( loadZstsExtension()==SQLITE_ERROR ){ ++ sqlite3_log(SQLITE_ERROR, "load zstd so failed :%s", dlerror()); ++ return SQLITE_ERROR; ++ } ++ }else{ ++ sqlite3_log(SQLITE_ERROR, "load compress so failed, compression is invalid :%u", compression); ++ return SQLITE_ERROR; ++ } ++#endif ++ return SQLITE_OK; ++} ++ ++/* Get compress bound */ ++static int compressLen(int src_len, int compression){ ++ if( compression==COMPRESSION_BROTLI || compression==COMPRESSION_ZSTD ){ ++ return compressBoundPtr(src_len); ++ } ++ return -1; ++} ++ ++/* Compress buf with compression */ ++static int compressBuf( ++ u8 *dst, ++ int dst_buf_len, ++ int *dst_written_len, ++ const u8 *src, ++ int src_len, ++ int compression ++){ ++ int ret_len = 0; ++ if( g_compress_algo_load==COMPRESSION_UNDEFINED && ++ loadCompressAlgorithmExtension(compression)==SQLITE_ERROR ){ ++ return SQLITE_ERROR; ++ } ++ if( g_compress_algo_load!=(u32)compression ){ ++ sqlite3_log(SQLITE_MISUSE, "already load %u, but need load %d", g_compress_algo_load, compression); ++ return SQLITE_MISUSE; ++ } ++ if( compression==COMPRESSION_BROTLI ){ ++ size_t dst_len = dst_buf_len; ++ int ret = brotliCompressPtr( ++ 3, // COMPRESS QUALITY (1-11) ++ BROTLI_MAX_WINDOW_BITS, // WINDOWS SIZE (10-24) ++ (size_t)src_len, ++ (const u8 *)src, ++ &dst_len, ++ (u8 *)dst); ++ if( ret!=BROTLI_TRUE ){ ++ return SQLITE_ERROR; ++ } ++ ret_len = dst_len; ++ }else if( compression==COMPRESSION_ZSTD ){ ++ ret_len = (int)zstdCompressPtr(dst, dst_buf_len, src, src_len, 3); ++ } ++ if( ret_len<=0 ){ ++ return SQLITE_ERROR; ++ } ++ *dst_written_len = ret_len; ++ return SQLITE_OK; ++} ++ ++/* Decompress buf with compression */ ++EXPORT_SYMBOLS static int decompressBuf( ++ u8 *dst, ++ int dst_buf_len, ++ int *dst_written_len, ++ const u8 *src, ++ int src_len, ++ int compression ++){ ++ int ret_len = -1; ++ if( g_compress_algo_load==COMPRESSION_UNDEFINED && ++ loadCompressAlgorithmExtension(compression)==SQLITE_ERROR ){ ++ return SQLITE_ERROR; ++ } ++ if( g_compress_algo_load!=(u32)compression ){ ++ sqlite3_log(SQLITE_MISUSE, "already load %u, but need load %d", g_compress_algo_load, compression); ++ return SQLITE_MISUSE; ++ } ++ if( compression==COMPRESSION_BROTLI ){ ++ size_t dst_len = dst_buf_len; ++ int ret = (int)brotliDecompressPtr(src_len, src, &dst_len, dst); ++ if( ret!=BROTLI_TRUE ){ ++ return SQLITE_ERROR; ++ } ++ ret_len = dst_len; ++ }else if( compression==COMPRESSION_ZSTD ){ ++ ret_len = (int)zstdDecompressPtr(dst, dst_buf_len, src, src_len); ++ } ++ if( ret_len<=0 ){ ++ return SQLITE_ERROR; ++ } ++ *dst_written_len = ret_len; ++ return SQLITE_OK; ++} ++ ++/* Check whether the table exists in the OutterDB. */ ++static int tableExists(sqlite3 *db, const char *table_name){ ++ sqlite3_stmt *stmt = NULL; ++ const char *sql = "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?;"; ++ int exists = 0; ++ ++ if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){ ++ sqlite3_bind_text(stmt, 1, table_name, -1, SQLITE_STATIC); ++ if( sqlite3_step(stmt)==SQLITE_ROW ){ ++ exists = 1; ++ } ++ sqlite3_finalize(stmt); ++ } ++ return exists; ++} ++ ++/* Get page size before compressed from OutterDB. */ ++static int getCompressPgsize(sqlite3 * db, int *pagesize){ ++ int rc = SQLITE_OK; ++ if( *pagesize!=0 ){ ++ return rc; ++ } ++ sqlite3_stmt *stmt = NULL; ++ const char *sql = "SELECT pagesize FROM vfs_compresssion;"; ++ if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){ ++ if( sqlite3_step(stmt)==SQLITE_ROW ){ ++ *pagesize = sqlite3_column_int(stmt, 0); ++ }else{ ++ rc = SQLITE_ERROR; ++ } ++ sqlite3_finalize(stmt); ++ } ++ return rc; ++} ++ ++/* Set page size before compressed to OutterDB. */ ++static int setCompressPgsize(sqlite3 *db, int pagesize){ ++ int rc = SQLITE_OK; ++ sqlite3_stmt *stmt = NULL; ++ const char *sql = "UPDATE vfs_compresssion SET pagesize=?;"; ++ ++ if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){ ++ sqlite3_bind_int(stmt, 1, pagesize); ++ if( sqlite3_step(stmt)!=SQLITE_DONE ){ ++ rc = SQLITE_ERROR; ++ } ++ sqlite3_finalize(stmt); ++ } ++ return rc; ++} ++ ++/* Get max page number from OutterDB. */ ++static int getMaxCompressPgno(sqlite3 * db){ ++ sqlite3_stmt *stmt = NULL; ++ const char *sql = "SELECT MAX(pageno) FROM vfs_pages;"; ++ int max_pgno = 0; ++ ++ if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){ ++ if( sqlite3_step(stmt)==SQLITE_ROW ){ ++ max_pgno = sqlite3_column_int(stmt, 0); ++ } ++ sqlite3_finalize(stmt); ++ } ++ return max_pgno; ++} ++ ++/* Get Compression option from OutterDB. */ ++static void getCompression(sqlite3 * db, CompressFile *pCompress){ ++ sqlite3_stmt *stmt = NULL; ++ const char *sql = "SELECT count(*), compression, pagesize FROM vfs_compression;"; ++ int count = 0; ++ pCompress->compression = COMPRESSION_UNDEFINED; ++ pCompress->page_size = 0; ++ ++ if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){ ++ if( sqlite3_step(stmt)==SQLITE_ROW ){ ++ count = sqlite3_column_int(stmt, 0); ++ if ( count==1 ){ ++ pCompress->compression = sqlite3_column_int(stmt, 1); ++ pCompress->page_size = sqlite3_column_int(stmt, 2); ++ } ++ } ++ sqlite3_finalize(stmt); ++ } ++ return; ++} ++ ++/* ++** Sync a compress file. If need commit a transaction ++** which begin in compressWrite or compressTruncate. ++*/ ++static int compressSync(sqlite3_file *pFile, int flags){ ++ assert( pFile ); ++ CompressFile *pCompress = (CompressFile *)pFile; ++ compress_info *compress_info = pCompress->pInfo; ++ sqlite3 *db = compress_info->pDb; ++ if( pCompress->bBegin==1 ){ ++ int rc = sqlite3_exec(db, "COMMIT;", NULL, NULL, NULL); ++ if( rc!=SQLITE_OK ){ ++ return SQLITE_IOERR_WRITE; ++ } ++ pCompress->bBegin = 0; ++ } ++ return SQLITE_OK; ++} ++ ++/* ++** Check if another file-handle holds a RESERVED lock on a compress file. ++*/ ++static int compressCheckReservedLock(sqlite3_file *pFile, int *pResOut){ ++ assert( pFile ); ++ *pResOut = 0; ++ return SQLITE_OK; ++} ++ ++/* ++** File control method. Use default VFS's xFileControl. ++*/ ++static int compressFileControl(sqlite3_file *pFile, int op, void *pArg){ ++ assert( pFile ); ++ pFile = ORIGFILE(pFile); ++ return pFile->pMethods->xFileControl(pFile, op, pArg); ++} ++ ++/* ++** Return the sector-size in bytes for a file. Use default VFS's xSectorSize. ++*/ ++static int compressSectorSize(sqlite3_file *pFile){ ++ assert( pFile ); ++ pFile = ORIGFILE(pFile); ++ return pFile->pMethods->xSectorSize(pFile); ++} ++ ++/* ++** Return the device characteristic flags supported by a file. ++** Use default VFS's xDeviceCharacteristics. ++*/ ++static int compressDeviceCharacteristics(sqlite3_file *pFile){ ++ assert( pFile ); ++ pFile = ORIGFILE(pFile); ++ return pFile->pMethods->xDeviceCharacteristics(pFile); ++} ++ ++/* ++** Lock a compress file.Never lock InnerDB, because it wil not control the database file. ++*/ ++static int compressLock(sqlite3_file *pFile, int eFileLock){ ++ assert( pFile ); ++ return SQLITE_OK; ++} ++ ++/* ++** Unlock a compress file.Never unlock InnerDB, because it wil not control the database file. ++*/ ++static int compressUnlock(sqlite3_file *pFile, int eFileLock){ ++ assert( pFile ); ++ return SQLITE_OK; ++} ++ ++/* ++** Get File's size. Here wil return InnerDB's file size. ++*/ ++static int compressFileSize(sqlite3_file *pFile, i64 *pSize){ ++ assert( pFile ); ++ CompressFile *pCompress = (CompressFile *)pFile; ++ compress_info *compress_info = pCompress->pInfo; ++ sqlite3 *db = compress_info->pDb; ++ int rc = getCompressPgsize(db, &pCompress->page_size); ++ if( rc!=SQLITE_OK ){ ++ return SQLITE_IOERR_FSTAT; ++ } ++ int pgsize = pCompress->page_size; ++ int maxpgno = getMaxCompressPgno(db); ++ *pSize = (i64)maxpgno * pgsize; ++ return SQLITE_OK; ++} ++ ++/* ++** Truncate a compress file by delete from vfs_pages. ++*/ ++static int compressTruncate(sqlite3_file *pFile, sqlite_int64 size){ ++ assert( pFile ); ++ CompressFile *pCompress = (CompressFile *)pFile; ++ compress_info *compress_info = pCompress->pInfo; ++ sqlite3 *db = compress_info->pDb; ++ int rc = getCompressPgsize(db, &pCompress->page_size); ++ if( rc!=SQLITE_OK || pCompress->page_size==0 ){ ++ return SQLITE_IOERR_TRUNCATE; ++ } ++ int pgsize = pCompress->page_size; ++ int pgno = size / pgsize; ++ if( size % pgsize!=0 || getMaxCompressPgno(db) < pgno){ ++ return SQLITE_IOERR_TRUNCATE; ++ } ++ if( pCompress->bBegin!=1 ){ ++ if( sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL)!=SQLITE_OK ){ ++ return SQLITE_IOERR_TRUNCATE; ++ } ++ pCompress->bBegin = 1; ++ } ++ sqlite3_stmt *stmt = NULL; ++ const char *sql = "DELETE FROM vfs_pages WHERE pageno > ?;"; ++ if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){ ++ sqlite3_bind_int(stmt, 1, pgno); ++ if( sqlite3_step(stmt)!=SQLITE_DONE ){ ++ rc = SQLITE_IOERR_TRUNCATE; ++ } ++ sqlite3_finalize(stmt); ++ } ++ return rc; ++} ++ ++/* ++** Write one page of data to compress file at a time. ++** It will be compressed and insert into vfs_pages in OutterDB. ++*/ ++static int compressWrite(sqlite3_file *pFile, const void *pBuf, int iAmt, sqlite_int64 iOfst){ ++ assert( pFile ); ++ assert( iAmt>0 ); ++ CompressFile *pCompress = (CompressFile *)pFile; ++ compress_info *compress_info = pCompress->pInfo; ++ sqlite3 *db = compress_info->pDb; ++ int rc = getCompressPgsize(db, &pCompress->page_size); ++ if( rc!=SQLITE_OK ){ ++ return SQLITE_IOERR_WRITE; ++ } ++ ++ if( pCompress->page_size<=0 && iAmt >= 512 && iAmt <= 64*1024 && !(iAmt & (iAmt - 1)) ){ ++ // new compress db need set orignal db's pagesize ++ rc = setCompressPgsize(db, iAmt); ++ if( rc!=SQLITE_OK ){ ++ return SQLITE_IOERR_WRITE; ++ } ++ pCompress->page_size = iAmt; ++ } ++ int pgsize = pCompress->page_size; ++ int pgno = iOfst / pgsize + 1; ++ if( iAmt!=pgsize || iOfst % pgsize!=0 ){ ++ return SQLITE_IOERR_WRITE; ++ } ++ ++ int max_compress_size = compressLen(iAmt, pCompress->compression); ++ if( max_compress_size<=0 ){ ++ return SQLITE_IOERR_WRITE; ++ } ++ u8 *compressed_data = sqlite3_malloc(max_compress_size); ++ if( compressed_data==0 ){ ++ return SQLITE_NOMEM; ++ } ++ int compress_data_len = 0; ++ if( compressBuf(compressed_data, max_compress_size, &compress_data_len, pBuf, iAmt, pCompress->compression) ){ ++ sqlite3_free(compressed_data); ++ return SQLITE_IOERR_WRITE; ++ } ++ if( pCompress->bBegin!=1 ){ ++ if( sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL)!=SQLITE_OK ){ ++ sqlite3_free(compressed_data); ++ return SQLITE_IOERR_WRITE; ++ } ++ pCompress->bBegin = 1; ++ } ++ sqlite3_stmt *stmt = NULL; ++ const char *sql = "INSERT OR REPLACE INTO vfs_pages(data, pageno) VALUE (?,?);"; ++ if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){ ++ sqlite3_bind_blob(stmt, 1, compressed_data, compress_data_len, SQLITE_STATIC); ++ sqlite3_bind_int(stmt, 2, pgno); ++ if( sqlite3_step(stmt)!=SQLITE_DONE ){ ++ sqlite3_free(compressed_data); ++ sqlite3_finalize(stmt); ++ return SQLITE_IOERR_WRITE; ++ } ++ sqlite3_finalize(stmt); ++ } ++ sqlite3_free(compressed_data); ++ return SQLITE_OK; ++} ++ ++/* ++** Read data from compress file. ++** It will be selected from vfs_pages in OutterDB and return a decompressed buf. ++*/ ++static int compressRead(sqlite3_file *pFile, void *pBuf, int iAmt, sqlite_int64 iOfst){ ++ assert( pFile ); ++ assert( iOfst>=0 ); ++ assert( iAmt>0 ); ++ CompressFile *pCompress = (CompressFile *)pFile; ++ if( pCompress->compression!=COMPRESSION_BROTLI && pCompress->compression!=COMPRESSION_ZSTD ){ ++ // nowaday only support brotli/zstd. ++ return SQLITE_CORRUPT; ++ } ++ (void)memset_s(pBuf, iAmt, 0, iAmt); ++ compress_info *compress_info = pCompress->pInfo; ++ sqlite3 *db = compress_info->pDb; ++ int rc = getCompressPgsize(db, &pCompress->page_size); ++ if( rc!=SQLITE_OK || pCompress->page_size==0 ){ ++ return SQLITE_IOERR_SHORT_READ; ++ } ++ int pgsize = pCompress->page_size; ++ int pgno = iOfst / pgsize + 1; ++ int dataidx = iOfst % pgsize; ++ sqlite3_stmt *stmt = NULL; ++ const char *sql = "SELECT data, length(data) FROM vfs_pages WHERE pageno=?;"; ++ const void *data = NULL; ++ int data_len = 0; ++ rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); ++ if( rc!=SQLITE_OK ){ ++ return SQLITE_CORRUPT; ++ } ++ u8 *decompressed_data = NULL; ++ if( pgsize!=iAmt ){ ++ decompressed_data = sqlite3_malloc(pgsize); ++ if( decompressed_data==NULL ){ ++ sqlite3_finalize(stmt); ++ return SQLITE_CORRUPT; ++ } ++ }else{ ++ decompressed_data = (u8 *)pBuf; ++ } ++ ++ int decompressed_data_len = 0; ++ sqlite3_bind_int(stmt, 1, pgno); ++ rc = sqlite3_step(stmt); ++ if( rc==SQLITE_ROW ){ ++ data = sqlite3_column_blob(stmt, 0); ++ if( data==NULL ){ ++ rc = SQLITE_IOERR_SHORT_READ; ++ goto failed; ++ } ++ data_len = sqlite3_column_int(stmt, 1); ++ if( data_len==0 ){ ++ rc = SQLITE_IOERR_SHORT_READ; ++ goto failed; ++ } ++ if( decompressBuf(decompressed_data, pgsize, &decompressed_data_len, data, data_len, ++ pCompress->compression)!=SQLITE_OK ){ ++ rc = SQLITE_IOERR_SHORT_READ; ++ goto failed; ++ } ++ if( decompressed_data_len!=pgsize ){ ++ rc = SQLITE_IOERR_SHORT_READ; ++ goto failed; ++ } ++ if( pgsize!=iAmt ){ ++ rc = memcpy_s(pBuf, iAmt, decompressed_data+dataidx, iAmt); ++ if( rc!=SQLITE_OK ){ ++ rc = SQLITE_IOERR_SHORT_READ; ++ goto failed; ++ } ++ } ++ rc = SQLITE_OK; ++ }else if( rc==SQLITE_DONE ){ ++ rc = SQLITE_IOERR_SHORT_READ; ++ } ++ ++ failed: ++ sqlite3_finalize(stmt); ++ if( pgsize!=iAmt ){ ++ sqlite3_free(decompressed_data); ++ } ++ ++ return rc; ++} ++ ++/* ++** Close compress file. Need sync data and close OutterDB and InnerDB. ++*/ ++static int compressClose(sqlite3_file *pFile){ ++ assert( pFile ); ++ CompressFile *pCompress = (CompressFile *)pFile; ++ compress_info *compress_info = pCompress->pInfo; ++ sqlite3 *db = compress_info->pDb; ++ int rc = compressSync(pFile, 0); ++ if( rc!=SQLITE_OK ){ ++ return rc; ++ } ++ if( compress_info->bOutterDbOpen ){ ++ rc = sqlite3_close(db); ++ if( rc!=SQLITE_OK ){ ++ return rc; ++ } ++ } ++ if( compress_info->bSubDbOpen ){ ++ pFile = ORIGFILE(pFile); ++ rc = pFile->pMethods->xClose(pFile); ++ } ++ return rc; ++} ++ ++/* ++** Create a shared memory file mapping. Need to rename the shm file, append a tail named "compress." ++*/ ++static int compressShmMap( ++ sqlite3_file *pFile, ++ int iPg, ++ int pgsz, ++ int fileFlag, ++ void volatile **pp ++){ ++ assert( pFile ); ++ pFile = ORIGFILE(pFile); ++ fileFlag |= SQLITE_OPEN_COMPRESS_SHM; ++ return pFile->pMethods->xShmMap(pFile, iPg, pgsz, fileFlag, pp); ++} ++ ++/* Perform locking on a shared-memory segment */ ++static int compressShmLock(sqlite3_file *pFile, int offset, int n, int flags){ ++ assert( pFile ); ++ pFile = ORIGFILE(pFile); ++ flags |= SQLITE_OPEN_COMPRESS_SHM; ++ return pFile->pMethods->xShmLock(pFile, offset, n, flags); ++} ++ ++/* Memory barrier operation on shared memory */ ++static void compressShmBarrier(sqlite3_file *pFile){ ++ assert( pFile ); ++ pFile = ORIGFILE(pFile); ++ return pFile->pMethods->xShmBarrier(pFile); ++} ++ ++/* Unmap a shared memory segment */ ++static int compressShmUnmap(sqlite3_file *pFile, int deleteFlag){ ++ assert( pFile ); ++ pFile = ORIGFILE(pFile); ++ deleteFlag |= SQLITE_OPEN_COMPRESS_SHM; ++ return pFile->pMethods->xShmUnmap(pFile, deleteFlag); ++} ++ ++/* Fetch a page of a memory-mapped file */ ++static int compressFetch(sqlite3_file *pFile, sqlite_int64 iOfst, int iAmt, void **pp){ ++ assert( pFile ); ++ pFile = ORIGFILE(pFile); ++ if( pFile->pMethods->iVersion>2 && pFile->pMethods->xFetch ){ ++ return pFile->pMethods->xFetch(pFile, iOfst, iAmt, pp); ++ } ++ *pp = 0; ++ return SQLITE_OK; ++} ++ ++/* Release a memory-mapped page */ ++static int compressUnfetch(sqlite3_file *pFile, sqlite_int64 iOfst, void *pPage){ ++ assert( pFile ); ++ pFile = ORIGFILE(pFile); ++ if( pFile->pMethods->iVersion>2 && pFile->pMethods->xUnfetch ){ ++ return pFile->pMethods->xUnfetch(pFile, iOfst, pPage); ++ } ++ return SQLITE_OK; ++} ++ ++/* Rename journal and wal file's name to identify InnerDB's file. */ ++static int compressGetJournalName(const char *zName, char **zJournalName){ ++ size_t len = strlen(zName); ++ if(( len >= 4 && strcmp(zName + len - 4, "-wal")==0 ) || ++ ( len >= 8 && strcmp(zName + len - 8, "-journal")==0 )){ ++ *zJournalName = (char *)sqlite3_malloc(strlen("compress") + strlen(zName) + 1); ++ if( *zJournalName==NULL ){ ++ return SQLITE_NOMEM; ++ } ++ strcpy(*zJournalName, zName); ++ strcpy(*zJournalName, "compress"); ++ } ++ return SQLITE_OK; ++} ++ ++/* ++** Open a compress file.If this file is not a journal or wal file, ++** it will open a OutterDB and create vfs_pages and vfs_compression table ++** which used to manager compressed pages's data. ++*/ ++static int compressOpen( ++ sqlite3_vfs *pVfs, ++ const char *zName, ++ sqlite3_file *pFile, ++ int flags, ++ int *pOutFlags ++){ ++ compress_info *pInfo = pVfs->pAppData; ++ sqlite3_file *pSubFile = ORIGFILE(pFile); ++ CompressFile *pCompress = (CompressFile *)pFile; ++ int rc = SQLITE_OK; ++ if( !(flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB)) ){ ++ if( flags & (SQLITE_OPEN_MAIN_JOURNAL|SQLITE_OPEN_WAL) ){ ++ flags |= SQLITE_OPEN_CREATE; ++ char *zJournalName = NULL; ++ rc = compressGetJournalName(zName, &zJournalName); ++ if( rc!=SQLITE_OK || zJournalName==NULL ){ ++ return rc; ++ } ++ rc = pInfo->pRootVfs->xOpen(pInfo->pRootVfs, zJournalName, pFile, flags, pOutFlags); ++ if( rc!=SQLITE_OK ){ ++ sqlite3_free(zJournalName); ++ } ++ return rc; ++ } ++ return pInfo->pRootVfs->xOpen(pInfo->pRootVfs, zName, pFile, flags, pOutFlags); ++ } ++ rc = pInfo->pRootVfs->xOpen(pInfo->pRootVfs, zName, pSubFile, flags, pOutFlags); ++ if( rc!=SQLITE_OK ){ ++ return rc; ++ } ++ pInfo->bSubDbOpen = 1; ++ sqlite3_int64 fileSize = 0; ++ rc = pSubFile->pMethods->xFileSize(pSubFile, &fileSize); ++ if( rc!=SQLITE_OK ){ ++ rc = SQLITE_CANTOPEN; ++ goto open_end; ++ } ++ ++ pCompress->pInfo = pInfo; ++ pFile->pMethods = &compress_io_methods; ++ rc = sqlite3_open_v2(zName, &pInfo->pDb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, pInfo->pRootVfs->zName); ++ if( rc!=SQLITE_OK ){ ++ rc = SQLITE_CANTOPEN; ++ goto open_end; ++ } ++ pInfo->bOutterDbOpen = 1; ++ sqlite3 *db = pInfo->pDb; ++ const char *pre_pragma = "PRAGMA page_size=4096;"\ ++ "PRAGMA auto_vacuum=INCREMENTAL;PRAGMA journal_mode=OFF;PRAGMA locking_mode=EXCLUSIVE;"; ++ rc = sqlite3_exec(db, pre_pragma, NULL, NULL, NULL); ++ if( rc!=SQLITE_OK ){ ++ rc = SQLITE_CANTOPEN; ++ goto open_end; ++ } ++ if( tableExists(db, "vfs_compression") ){ ++ getCompression(db, pCompress); ++ if( pCompress->compression!=COMPRESSION_BROTLI && pCompress->compression!=COMPRESSION_ZSTD ){ ++ rc = SQLITE_CANTOPEN; ++ goto open_end; ++ } ++ if( loadCompressAlgorithmExtension(pCompress->compression)==SQLITE_ERROR ){ ++ rc = SQLITE_CANTOPEN; ++ goto open_end; ++ } ++ }else if( flags&SQLITE_OPEN_MAIN_DB && fileSize!=0){ ++ rc = SQLITE_WARNING_NOTCOMPRESSDB; ++ sqlite3_log(rc, "open compress database go wrong, it should be a compressed db"); ++ goto open_end; ++ }else{ ++ if( loadCompressAlgorithmExtension(COMPRESSION_UNDEFINED)==SQLITE_ERROR ){ ++ rc = SQLITE_CANTOPEN; ++ goto open_end; ++ } ++ const char *init_compression_sql = "CREATE TABLE vfs_compression (compression INTEGER, pagesize INTEGER);"; ++ rc = sqlite3_exec(db, init_compression_sql, NULL, NULL, NULL); ++ if( rc!=SQLITE_OK ){ ++ rc = SQLITE_CANTOPEN; ++ goto open_end; ++ } ++ char set_compression_sql[COMPRESSION_SQL_MAX_LENGTH] = {0}; ++ if( sprintf_s(set_compression_sql, COMPRESSION_SQL_MAX_LENGTH, ++ "INSERT INTO vfs_compression(compression, pagesize) VALUE (%u, 0);", g_compress_algo_load)<=0 ){ ++ rc = SQLITE_CANTOPEN; ++ goto open_end; ++ } ++ rc = sqlite3_exec(db, set_compression_sql, NULL, NULL, NULL); ++ if( rc!=SQLITE_OK ){ ++ rc = SQLITE_CANTOPEN; ++ goto open_end; ++ } ++ pCompress->compression = g_compress_algo_load; ++ pCompress->page_size = 0; ++ } ++ if( tableExists(db, "vfs_pages") ){ ++ goto open_end; ++ } ++ const char *init_page_sql = "CREATE TABLE vfs_pages (pgno INTEGER PRIMARY KEY, data BLOB NOT NULL);"; ++ rc = sqlite3_exec(db, init_page_sql, NULL, NULL, NULL); ++ if( rc!=SQLITE_OK ){ ++ rc = SQLITE_CANTOPEN; ++ } ++ ++ open_end: ++ return rc; ++} ++ ++/* ++** Delete compress file. If this file is journal or wal, need to delete the InnerDB's renamed file. ++*/ ++static int compressDelete(sqlite3_vfs *pVfs, const char *zName, int syncDir){ ++ compress_info *pInfo = pVfs->pAppData; ++ char *zJournalName = NULL; ++ int rc = compressGetJournalName(zName, &zJournalName); ++ if( rc!=SQLITE_OK ){ ++ return rc; ++ } ++ if( zJournalName!=NULL ){ ++ rc = pInfo->pRootVfs->xDelete(pInfo->pRootVfs, zJournalName, syncDir); ++ sqlite3_free(zJournalName); ++ return rc; ++ }else{ ++ return pInfo->pRootVfs->xDelete(pInfo->pRootVfs, zName, syncDir); ++ } ++} ++ ++/* ++** Access compress file. If this file is journal or wal need to access the InnerDB's renamed file. ++*/ ++static int compressAccess( ++ sqlite3_vfs *pVfs, ++ const char *zName, ++ int flags, ++ int *pResOut ++){ ++ compress_info *pInfo = pVfs->pAppData; ++ char *zJournalName = NULL; ++ int rc = compressGetJournalName(zName, &zJournalName); ++ if( rc!=SQLITE_OK ){ ++ return rc; ++ } ++ if( zJournalName!=NULL ){ ++ rc = pInfo->pRootVfs->xAccess(pInfo->pRootVfs, zJournalName, flags, pResOut); ++ sqlite3_free(zJournalName); ++ return rc; ++ }else{ ++ return pInfo->pRootVfs->xAccess(pInfo->pRootVfs, zName, flags, pResOut); ++ } ++} ++ ++ ++/* ++** Init compressvfs and register the vfs ++*/ ++EXPORT_SYMBOLS int sqlite3CompressvfsInit(){ ++ // copy default vfs for compressvfs ++ if( sqlite3_vfs_find(COMPRESS_VFS_NAME) ){ ++ return SQLITE_OK; ++ } ++ sqlite3_vfs *default_vfs = sqlite3_vfs_find(0); ++ if( default_vfs==0 ){ ++ sqlite3_log(SQLITE_ERROR, "Default vfs not found!"); ++ return SQLITE_ERROR; ++ } ++ sqlite3_vfs *pVfs = &compress_vfs; ++ compress_info *pInfo = &compress_vfs_info; ++ *pVfs = *default_vfs; ++ pVfs->pNext = 0; ++ pVfs->zName = COMPRESS_VFS_NAME; ++ pVfs->pAppData = pInfo; ++ pVfs->szOsFile = pVfs->szOsFile+sizeof(CompressFile); ++ ++ pVfs->xOpen = compressOpen; ++ pVfs->xDelete = compressDelete; ++ pVfs->xAccess = compressAccess; ++ pInfo->pRootVfs = default_vfs; ++ pInfo->bSubDbOpen = 0; ++ pInfo->bOutterDbOpen = 0; ++ int rc = sqlite3_vfs_register(pVfs, 0); ++ if( rc!=SQLITE_OK ){ ++ sqlite3_log(SQLITE_ERROR, "Compress vfs register error!"); ++ return rc; ++ } ++ return SQLITE_OK; ++} ++ ++EXPORT_SYMBOLS sqlite3_file *compressvfsGetOrigFile(sqlite3_file *file){ ++ return ORIGFILE(file); ++} ++ ++#endif /* SQLITE_ENABLE_PAGE_COMPRESS */ +\ No newline at end of file +diff --git a/src/sqlite3.c b/src/sqlite3.c +index 24e3247..67a78e3 100644 +--- a/src/sqlite3.c ++++ b/src/sqlite3.c +@@ -881,6 +881,7 @@ SQLITE_API int sqlite3_exec( + #define SQLITE_NOTICE_RBU (SQLITE_NOTICE | (3<<8)) + #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) + #define SQLITE_WARNING_DUMP (SQLITE_WARNING | (2<<8)) ++#define SQLITE_WARNING_NOTCOMPRESSDB (SQLITE_WARNING | (3<<8)) + #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) + #define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) + #define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal use only */ +@@ -1877,6 +1878,14 @@ struct sqlite3_vfs { + */ + #define SQLITE_SHM_NLOCK 8 + ++/* ++** CAPI3REF: Maximum xShmLock index ++** ++** The xShmMap method on [sqlite3_io_methods] may use values ++** between 0 and this upper bound as its "offset" argument. ++*/ ++#define SQLITE_SHMMAP_IS_WRITE 0x00000001 /*Flag for xShmMap, extend file if necessary */ ++#define SQLITE_OPEN_COMPRESS_SHM 0x00010000 /*Flag for xShmMap, need to rename shm file */ + + /* + ** CAPI3REF: Initialize The SQLite Library +@@ -38639,6 +38648,9 @@ struct unixFile { + UnixUnusedFd *pPreallocatedUnused; /* Pre-allocated UnixUnusedFd */ + const char *zPath; /* Name of the file */ + unixShm *pShm; /* Shared memory segment information */ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ unixShm *pCompressShm; ++#endif + int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ + #if SQLITE_MAX_MMAP_SIZE>0 + int nFetchOut; /* Number of outstanding xFetch refs */ +@@ -39640,6 +39652,9 @@ struct unixInodeInfo { + UnixUnusedFd *pUnused; /* Unused file descriptors to close */ + int nRef; /* Number of pointers to this structure */ + unixShmNode *pShmNode; /* Shared memory associated with this inode */ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ unixShmNode *pCompressShmNode; ++#endif + unixInodeInfo *pNext; /* List of all unixInodeInfo objects */ + unixInodeInfo *pPrev; /* .... doubly linked */ + #if SQLITE_ENABLE_LOCKING_STYLE +@@ -40602,6 +40617,12 @@ static int closeUnixFile(sqlite3_file *id){ + OSTRACE(("CLOSE %-3d\n", pFile->h)); + OpenCounter(-1); + sqlite3_free(pFile->pPreallocatedUnused); ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ if( strcmp(pFile->zPath+strlen(pFile->zPath)-strlen("compress"), "compress")==0 ){ ++ sqlite3_free(*(char**)&pFile->zPath); ++ pFile->zPath = 0; ++ } ++#endif + memset(pFile, 0, sizeof(unixFile)); + return SQLITE_OK; + } +@@ -42889,14 +42910,23 @@ static int unixShmSystemLock( + unixFile *pFile, /* Open connection to the WAL file */ + int lockType, /* F_UNLCK, F_RDLCK, or F_WRLCK */ + int ofst, /* First byte of the locking range */ +- int n /* Number of bytes to lock */ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ int n, /* Number of bytes to lock */ ++ int bCompress /* True to compress shm */ ++#else ++ int n ++#endif + ){ + unixShmNode *pShmNode; /* Apply locks to this open shared-memory segment */ + struct flock f; /* The posix advisory locking structure */ + int rc = SQLITE_OK; /* Result code form fcntl() */ + + /* Access to the unixShmNode object is serialized by the caller */ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ pShmNode = bCompress ? pFile->pInode->pCompressShmNode : pFile->pInode->pShmNode; ++#else + pShmNode = pFile->pInode->pShmNode; ++#endif + assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->pShmMutex) ); + assert( pShmNode->nRef>0 || unixMutexHeld() ); + +@@ -42984,9 +43014,11 @@ static int unixShmRegionPerMap(void){ + ** This is not a VFS shared-memory method; it is a utility function called + ** by VFS shared-memory methods. + */ +-static void unixShmPurge(unixFile *pFd){ +- unixShmNode *p = pFd->pInode->pShmNode; +- assert( unixMutexHeld() ); ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++static void unixShmPurgeInner(unixFile *pFd, unixShmNode *p, int bCompressShm){ ++#else ++static void unixShmPurgeInner(unixFile *pFd, unixShmNode *p){ ++#endif + if( p && ALWAYS(p->nRef==0) ){ + int nShmPerMap = unixShmRegionPerMap(); + int i; +@@ -43004,11 +43036,35 @@ static void unixShmPurge(unixFile *pFd){ + robust_close(pFd, p->hShm, __LINE__); + p->hShm = -1; + } ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ if( bCompressShm ){ ++ p->pInode->pCompressShmNode = 0; ++ }else{ ++ p->pInode->pShmNode = 0; ++ } ++#else + p->pInode->pShmNode = 0; ++#endif + sqlite3_free(p); + } + } + ++/* ++** Purge the unixShmNodeList list of all entries with unixShmNode.nRef==0. ++** ++** This is not a VFS shared-memory method; it is a utility function called ++** by VFS shared-memory methods. ++*/ ++static void unixShmPurge(unixFile *pFd){ ++ assert( unixMutexHeld() ); ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ unixShmPurgeInner(pFd, pFd->pInode->pCompressShmNode, 1); ++ unixShmPurgeInner(pFd, pFd->pInode->pShmNode, 0); ++#else ++ unixShmPurgeInner(pFd, pFd->pInode->pShmNode); ++#endif ++} ++ + /* + ** The DMS lock has not yet been taken on shm file pShmNode. Attempt to + ** take it now. Return SQLITE_OK if successful, or an SQLite error +@@ -43018,7 +43074,11 @@ static void unixShmPurge(unixFile *pFd){ + ** connection and no other process already holds a lock, return + ** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. + */ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode, int bCompress){ ++#else + static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ ++#endif + struct flock lock; + int rc = SQLITE_OK; + +@@ -43051,7 +43111,11 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ + pShmNode->isUnlocked = 1; + rc = SQLITE_READONLY_CANTINIT; + }else{ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1, bCompress); ++#else + rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); ++#endif + MarkLockStatusByRc(rc, SHM_DMS_IDX, 1, EXCLUSIVE_LOCK, LOCK_BY_PROCESS); + MARK_LAST_BUSY_LINE(rc); + /* The first connection to attach must truncate the -shm file. We +@@ -43071,7 +43135,11 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ + + if( rc==SQLITE_OK ){ + assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK ); ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1, bCompress); ++#else + rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); ++#endif + MarkLockStatusByRc(rc, SHM_DMS_IDX, 1, SHARED_LOCK, LOCK_BY_PROCESS); + MARK_LAST_BUSY_LINE(rc); + } +@@ -43113,7 +43181,11 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ + ** that case, we do not really need shared memory. No shared memory + ** file is created. The shared memory will be simulated with heap memory. + */ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++static int unixOpenSharedMemory(unixFile *pDbFd, int bCompressShm){ ++#else + static int unixOpenSharedMemory(unixFile *pDbFd){ ++#endif + struct unixShm *p = 0; /* The connection to be opened */ + struct unixShmNode *pShmNode; /* The underlying mmapped file */ + int rc = SQLITE_OK; /* Result code */ +@@ -43133,7 +43205,11 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ + assert( unixFileMutexNotheld(pDbFd) ); + unixEnterMutex(); + pInode = pDbFd->pInode; ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ pShmNode = bCompressShm ? pInode->pCompressShmNode : pInode->pShmNode; ++#else + pShmNode = pInode->pShmNode; ++#endif + if( pShmNode==0 ){ + struct stat sStat; /* fstat() info for database file */ + #ifndef SQLITE_SHM_DIRECTORY +@@ -43153,6 +43229,11 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ + nShmFilename = sizeof(SQLITE_SHM_DIRECTORY) + 31; + #else + nShmFilename = 6 + (int)strlen(zBasePath); ++#endif ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ if( bCompressShm ){ ++ nShmFilename += 8; ++ } + #endif + pShmNode = sqlite3_malloc64( sizeof(*pShmNode) + nShmFilename ); + if( pShmNode==0 ){ +@@ -43169,8 +43250,17 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ + sqlite3_snprintf(nShmFilename, zShm, "%s-shm", zBasePath); + sqlite3FileSuffix3(pDbFd->zPath, zShm); + #endif +- pShmNode->hShm = -1; ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ if( bCompressShm ){ ++ sqlite3_snprintf(nShmFilename, zShm, "%scompress", zShm); ++ pDbFd->pInode->pCompressShmNode = pShmNode; ++ }else{ ++ pDbFd->pInode->pShmNode = pShmNode; ++ } ++#else + pDbFd->pInode->pShmNode = pShmNode; ++#endif ++ pShmNode->hShm = -1; + pShmNode->pInode = pDbFd->pInode; + if( sqlite3GlobalConfig.bCoreMutex ){ + pShmNode->pShmMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); +@@ -43207,7 +43297,11 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ + */ + robustFchown(pShmNode->hShm, sStat.st_uid, sStat.st_gid); + ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ rc = unixLockSharedMemory(pDbFd, pShmNode, bCompressShm); ++#else + rc = unixLockSharedMemory(pDbFd, pShmNode); ++#endif + if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; + } + } +@@ -43218,7 +43312,15 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ + p->id = pShmNode->nextShmId++; + #endif + pShmNode->nRef++; ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ if( bCompressShm ){ ++ pDbFd->pCompressShm = p; ++ }else{ ++ pDbFd->pShm = p; ++ } ++#else + pDbFd->pShm = p; ++#endif + unixLeaveMutex(); + + /* The reference count on pShmNode has already been incremented under +@@ -43250,6 +43352,10 @@ shm_open_err: + ** + ** If an error occurs, an error code is returned and *pp is set to NULL. + ** ++** The fileFlag parameter has two means: ++** bExtend = fileFlag & SQLITE_SHMMAP_IS_WRITE ++** bCompressShm = fileFlag & SQLITE_OPEN_COMPRESS_SHM ++** + ** Otherwise, if the bExtend parameter is 0 and the requested shared-memory + ** region has not been allocated (by any client, including one running in a + ** separate process), then *pp is set to NULL and SQLITE_OK returned. If +@@ -43260,12 +43366,15 @@ shm_open_err: + ** this call as described above, then it is mapped into this processes + ** address space (if it is not already), *pp is set to point to the mapped + ** memory and SQLITE_OK returned. ++** ++** If bCompressShm is true, it indicates that the shmFile needs to be renamed by ++** adding a suffix named "compress" + */ + static int unixShmMap( + sqlite3_file *fd, /* Handle open on database file */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ +- int bExtend, /* True to extend file if necessary */ ++ int fileFlag, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ + ){ + unixFile *pDbFd = (unixFile*)fd; +@@ -43274,18 +43383,35 @@ static int unixShmMap( + int rc = SQLITE_OK; + int nShmPerMap = unixShmRegionPerMap(); + int nReqRegion; ++ int bExtend = (fileFlag & SQLITE_SHMMAP_IS_WRITE); // True to extend file if necessary ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ int bCompressShm = (fileFlag & SQLITE_OPEN_COMPRESS_SHM); // True to rename shm file ++#endif + + /* If the shared-memory file has not yet been opened, open it now. */ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ if(( bCompressShm && pDbFd->pCompressShm==0 ) || ( !bCompressShm && pDbFd->pShm==0 )){ ++ rc = unixOpenSharedMemory(pDbFd, bCompressShm); ++ if( rc!=SQLITE_OK ) return rc; ++ } ++ ++ p = bCompressShm ? pDbFd->pCompressShm : pDbFd->pShm; ++#else + if( pDbFd->pShm==0 ){ + rc = unixOpenSharedMemory(pDbFd); + if( rc!=SQLITE_OK ) return rc; + } + + p = pDbFd->pShm; ++#endif + pShmNode = p->pShmNode; + sqlite3_mutex_enter(pShmNode->pShmMutex); + if( pShmNode->isUnlocked ){ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ rc = unixLockSharedMemory(pDbFd, pShmNode, bCompressShm); ++#else + rc = unixLockSharedMemory(pDbFd, pShmNode); ++#endif + if( rc!=SQLITE_OK ) goto shmpage_out; + pShmNode->isUnlocked = 0; + } +@@ -43448,8 +43574,13 @@ static int unixShmLock( + int rc = SQLITE_OK; /* Result code */ + u16 mask; /* Mask of locks to take or release */ + int *aLock; +- ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ int bCompress = (flags & SQLITE_OPEN_COMPRESS_SHM); ++ flags &= ~SQLITE_OPEN_COMPRESS_SHM; ++ p = bCompress ? pDbFd->pCompressShm : pDbFd->pShm; ++#else + p = pDbFd->pShm; ++#endif + if( p==0 ) { + #ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR_SHMLOCK, "unixShmLock-pShm, fd[%d], ofst[%d], n[%d], flags[%d]", pDbFd->h, ofst, n, flags); +@@ -43516,7 +43647,11 @@ static int unixShmLock( + } + + if( bUnlock ){ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n, bCompress); ++#else + rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n); ++#endif + if( rc==SQLITE_OK ){ + memset(&aLock[ofst], 0, sizeof(int)*n); + } +@@ -43540,7 +43675,11 @@ static int unixShmLock( + if( aLock[ofst]<0 ){ + rc = SQLITE_BUSY; + }else if( aLock[ofst]==0 ){ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n, bCompress); ++#else + rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n); ++#endif + useProcessLock = LOCK_BY_PROCESS; + } + MarkLockStatusByRc(rc, ofst, n, SHARED_LOCK, useProcessLock); +@@ -43566,7 +43705,11 @@ static int unixShmLock( + /* Get the exclusive locks at the system level. Then if successful + ** also update the in-memory values. */ + if( rc==SQLITE_OK ){ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ rc = unixShmSystemLock(pDbFd, F_WRLCK, ofst+UNIX_SHM_BASE, n, bCompress); ++#else + rc = unixShmSystemLock(pDbFd, F_WRLCK, ofst+UNIX_SHM_BASE, n); ++#endif + useProcessLock = LOCK_BY_PROCESS; + if( rc==SQLITE_OK ){ + assert( (p->sharedMask & mask)==0 ); +@@ -43619,9 +43762,14 @@ static int unixShmUnmap( + unixShmNode *pShmNode; /* The underlying shared-memory file */ + unixShm **pp; /* For looping over sibling connections */ + unixFile *pDbFd; /* The underlying database file */ +- + pDbFd = (unixFile*)fd; ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ int bCompressShm = (deleteFlag & SQLITE_OPEN_COMPRESS_SHM); // True means compress shm file ++ deleteFlag &= 1; ++ p = bCompressShm ? pDbFd->pCompressShm : pDbFd->pShm; ++#else + p = pDbFd->pShm; ++#endif + if( p==0 ) return SQLITE_OK; + pShmNode = p->pShmNode; + +@@ -43636,7 +43784,15 @@ static int unixShmUnmap( + + /* Free the connection p */ + sqlite3_free(p); ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ if( bCompressShm ){ ++ pDbFd->pCompressShm = 0; ++ }else{ ++ pDbFd->pShm = 0; ++ } ++#else + pDbFd->pShm = 0; ++#endif + sqlite3_mutex_leave(pShmNode->pShmMutex); + + /* If pShmNode->nRef has reached 0, then close the underlying +@@ -50788,8 +50944,15 @@ static int winLockSharedMemory(winShmNode *pShmNode){ + ** When opening a new shared-memory file, if no other instances of that + ** file are currently open, in this process or in other processes, then + ** the file must be truncated to zero length or have its header cleared. ++** ++** If bCompressShm is true, it indicates that the shmFile needs to be renamed by ++** adding a suffix named "compress" + */ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++static int winOpenSharedMemory(winFile *pDbFd, int bCompressShm){ ++#else + static int winOpenSharedMemory(winFile *pDbFd){ ++#endif + struct winShm *p; /* The connection to be opened */ + winShmNode *pShmNode = 0; /* The underlying mmapped file */ + int rc = SQLITE_OK; /* Result code */ +@@ -50804,6 +50967,11 @@ static int winOpenSharedMemory(winFile *pDbFd){ + p = sqlite3MallocZero( sizeof(*p) ); + if( p==0 ) return SQLITE_IOERR_NOMEM_BKPT; + nName = sqlite3Strlen30(pDbFd->zPath); ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ if( bCompressShm ){ ++ nName += 8; ++ } ++#endif + pNew = sqlite3MallocZero( sizeof(*pShmNode) + nName + 17 ); + if( pNew==0 ){ + sqlite3_free(p); +@@ -50812,6 +50980,11 @@ static int winOpenSharedMemory(winFile *pDbFd){ + pNew->zFilename = (char*)&pNew[1]; + sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); + sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ if( bCompressShm ){ ++ sqlite3_snprintf(nName+15, pNew->zFilename, "%scompress", pNew->zFilename); ++ } ++#endif + + /* Look to see if there is an existing winShmNode that can be used. + ** If no matching winShmNode currently exists, create a new one. +@@ -51069,6 +51242,10 @@ static void winShmBarrier( + ** + ** If an error occurs, an error code is returned and *pp is set to NULL. + ** ++** The fileFlag parameter has two means: ++** bExtend = fileFlag & SQLITE_SHMMAP_IS_WRITE ++** bCompressShm = fileFlag & SQLITE_OPEN_COMPRESS_SHM ++** + ** Otherwise, if the isWrite parameter is 0 and the requested shared-memory + ** region has not been allocated (by any client, including one running in a + ** separate process), then *pp is set to NULL and SQLITE_OK returned. If +@@ -51079,12 +51256,15 @@ static void winShmBarrier( + ** this call as described above, then it is mapped into this processes + ** address space (if it is not already), *pp is set to point to the mapped + ** memory and SQLITE_OK returned. ++** ++** If bCompressShm is true, it indicates that the shmFile needs to be renamed by ++** adding a suffix named "compress" + */ + static int winShmMap( + sqlite3_file *fd, /* Handle open on database file */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ +- int isWrite, /* True to extend file if necessary */ ++ int fileFlag, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ + ){ + winFile *pDbFd = (winFile*)fd; +@@ -51092,10 +51272,18 @@ static int winShmMap( + winShmNode *pShmNode; + DWORD protect = PAGE_READWRITE; + DWORD flags = FILE_MAP_WRITE | FILE_MAP_READ; ++ int isWrite = (fileFlag & SQLITE_SHMMAP_IS_WRITE); // True to extend file if necessary ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ int bCompressShm = (fileFlag & SQLITE_OPEN_COMPRESS_SHM); // True to rename shm file ++#endif + int rc = SQLITE_OK; + + if( !pShm ){ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ rc = winOpenSharedMemory(pDbFd, bCompressShm); ++#else + rc = winOpenSharedMemory(pDbFd); ++#endif + if( rc!=SQLITE_OK ) return rc; + pShm = pDbFd->pShm; + assert( pShm!=0 ); +@@ -66177,6 +66365,7 @@ static SQLITE_NOINLINE int walIndexPageRealloc( + pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); + if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT; + }else{ ++ assert( pWal->writeLock==0 || pWal->writeLock==1 ); + rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, + pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] + ); +@@ -137863,6 +138052,9 @@ typedef int (*sqlite3_loadext_entry)( + /* clean the binlog of the db */ + #define sqlite3_clean_binlog sqlite3_api->clean_binlog + #endif /* SQLITE_ENABLE_BINLOG */ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++#define sqlite3_compressdb_backup sqlite3_api->compressdb_backup ++#endif /* SQLITE_ENABLE_PAGE_COMPRESS */ + #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ + + #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) +@@ -183070,6 +183262,188 @@ SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ + return oldLimit; /* IMP: R-53341-35419 */ + } + ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++/************** Include sqlitecompressvfs.h in the middle of main.c ******************/ ++/************** Begin file sqlitecompressvfs.h ***************************************/ ++/* ++** 2025 June 27 ++** ++** The author disclaims copyright to this source code. In place of ++** a legal notice, here is a blessing: ++** ++** May you do good and not evil. ++** May you find forgiveness for yourself and forgive others. ++** May you share freely, never taking more than you give. ++** ++****************************************************************************** ++** ++** This header file is used by programs that want to link against the ++** compressvfs extension. All it does is declare the sqlite3_compressvfs_init() interface. ++*/ ++ ++#ifndef _WIN32 ++#include ++#endif ++ ++typedef int (*sqlite3CompressVFSInit_ptr)(); ++static sqlite3CompressVFSInit_ptr compressvfsInitPtr = NULL; ++typedef int (*sqlite3DecompressBuf_ptr)( ++ u8 *dst, ++ int dst_buf_len, ++ int *dst_written_len, ++ const u8 *src, ++ int src_len, ++ int compression ++); ++static sqlite3DecompressBuf_ptr decompressBufPtr = NULL; ++typedef sqlite3_file *(*sqlite3CompressGetOriFile_ptr)(sqlite3_file *); ++static sqlite3CompressGetOriFile_ptr compressvfsGetOrigFilePtr = NULL; ++static u32 compressInit = 0u; ++static u32 compressSoLoad = 0u; ++static void *g_compress_library = NULL; ++ ++int sqlite3LoadCompressExtension(){ ++ if( compressSoLoad ){ ++ return SQLITE_OK; ++ } ++#ifndef _WIN32 ++ g_compress_library = dlopen("libsqlitecompressvfs.z.so", RTLD_LAZY); ++ if( g_compress_library==NULL ){ ++ sqlite3_log(SQLITE_ERROR, "load compressvfs so failed: %s\n", dlerror()); ++ return SQLITE_ERROR; ++ } ++ compressvfsInitPtr = (sqlite3CompressVFSInit_ptr)dlsym(g_compress_library, "sqlite3CompressvfsInit"); ++ if( compressvfsInitPtr==NULL ){ ++ sqlite3_log(SQLITE_ERROR, "load compressvfs init func failed: %s\n", dlerror()); ++ dlclose(g_compress_library); ++ return SQLITE_ERROR; ++ } ++ compressvfsGetOrigFilePtr = (sqlite3CompressGetOriFile_ptr)dlsym(g_compress_library, "compressvfsGetOrigFile"); ++ if( compressvfsGetOrigFilePtr==NULL ){ ++ sqlite3_log(SQLITE_ERROR, "load compressvfs get orig file func failed: %s\n", dlerror()); ++ dlclose(g_compress_library); ++ return SQLITE_ERROR; ++ } ++ decompressBufPtr = (sqlite3DecompressBuf_ptr)dlsym(g_compress_library, "decompressBuf"); ++ if( decompressBufPtr==NULL ){ ++ sqlite3_log(SQLITE_ERROR, "load decompress func failed: %s\n", dlerror()); ++ dlclose(g_compress_library); ++ return SQLITE_ERROR; ++ } ++ compressSoLoad = 1u; ++#endif ++ return SQLITE_OK; ++} ++ ++/* ++** Load compressvfs so and register compressvfs ++*/ ++int sqlite3CompressVFSModuleInit(){ ++ int rc = SQLITE_OK; ++ if( compressInit ){ ++ return rc; ++ } ++#ifndef _WIN32 ++ rc = sqlite3LoadCompressExtension(); ++ if( rc!=SQLITE_OK ){ ++ return rc; ++ } ++ ++ rc = compressvfsInitPtr(); ++ if( rc!=SQLITE_OK ){ ++ dlclose(g_compress_library); ++ compressSoLoad = 0u; ++ return rc; ++ } ++ compressInit = 1u; ++#endif ++ return rc; ++} ++ ++/* ++** The backup API copies the content of one compressed database into another decompressed file. ++** It is useful either for creating backups database or ++** for copying in-memory databases to or from persistent files. ++** The source database must be comrpessed, the destination database will be decomrpessed. ++** IMPORTANT: Before use this API, must use a normal vfs which is not compressvfs to open source comrpessed db. ++*/ ++SQLITE_API int sqlite3_compressdb_backup(sqlite3 *srcDb, const char *destDbPath){ ++ int rc = sqlite3LoadCompressExtension(); ++ if( rc!=SQLITE_OK ){ ++ return rc; ++ } ++ int pagesize = 0; ++ int compression = 0; ++ sqlite3_stmt *stmt = NULL; ++ u8 *decompressed_data = NULL; ++ int openFlags = (O_RDWR|O_CREAT|O_LARGEFILE|O_BINARY|O_NOFOLLOW); ++ int fd = robust_open(destDbPath, openFlags, 0); ++ if( fd<0 ){ ++ sqlite3_log(SQLITE_ERROR, "Failed to open destDb file!"); ++ return SQLITE_CANTOPEN_BKPT; ++ } ++ const char *compression_sql = "SELECT pagesize, compression FROM vfs_compression;"; ++ if( sqlite3_prepare_v2(srcDb, compression_sql, -1, &stmt, NULL) == SQLITE_OK ){ ++ if( sqlite3_step(stmt) == SQLITE_ROW ){ ++ pagesize = sqlite3_column_int(stmt, 0); ++ compression = sqlite3_column_int(stmt, 1); ++ }else{ ++ sqlite3_log(SQLITE_ERROR, "Failed to select compression!"); ++ rc = SQLITE_ERROR; ++ } ++ sqlite3_finalize(stmt); ++ stmt = NULL; ++ } ++ if( rc!=SQLITE_OK || pagesize==0 ){ ++ rc = SQLITE_WARNING_NOTCOMPRESSDB; ++ goto failed; ++ } ++ int dst_len = pagesize; ++ const char *data_sql = "SELECT data FROM vfs_pages;"; ++ rc = sqlite3_prepare_v2(srcDb, data_sql, -1, &stmt, NULL); ++ if( rc!=SQLITE_OK ){ ++ sqlite3_log(SQLITE_ERROR, "Failed to exec data_sql!"); ++ goto failed; ++ } ++ decompressed_data = (u8 *)sqlite3_malloc(pagesize); ++ int pagecount = 0; ++ while( (rc = sqlite3_step(stmt)) == SQLITE_ROW ){ ++ const void *data_ptr = sqlite3_column_blob(stmt, 0); ++ int data_size = sqlite3_column_bytes(stmt, 0); ++ rc = decompressBufPtr(decompressed_data, pagesize, &dst_len, data_ptr, data_size, compression); ++ if( rc!=SQLITE_OK || pagesize!=dst_len ){ ++ sqlite3_log(rc, "Failed to decompress buf in src db!"); ++ rc = SQLITE_ERROR; ++ goto failed; ++ } ++ int nWrite = seekAndWriteFd(fd, (i64)pagecount*pagesize, decompressed_data, pagesize, &rc); ++ if( nWrite!=dst_len || rc!=SQLITE_OK ){ ++ sqlite3_log(rc, "Write dest db error at page %d!", pagecount); ++ rc = SQLITE_IOERR_WRITE; ++ goto failed; ++ } ++ pagecount++; ++ if( pagecount==(PENDING_BYTE/pagesize)+1 ){ ++ pagecount++; ++ } ++ } ++ if( rc!=SQLITE_DONE ){ ++ sqlite3_log(rc, "Querry error: %s!", sqlite3_errmsg(srcDb)); ++ } ++ ++failed: ++ osClose(fd); ++ if( decompressed_data ){ ++ sqlite3_free(decompressed_data); ++ } ++ sqlite3_finalize(stmt); ++ return rc; ++} ++ ++/************** End of sqlitecompressvfs.h ***************************************/ ++/************** Continuing where we left off in main.c ***********************/ ++#endif /* SQLITE_ENABLE_PAGE_COMPRESS */ ++ + /* + ** This function is used to parse both URIs and non-URI filenames passed by the + ** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database +@@ -183314,6 +183688,17 @@ SQLITE_PRIVATE int sqlite3ParseUri( + flags &= ~SQLITE_OPEN_URI; + } + ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ if( sqlite3_stricmp(zVfs, "compressvfs")==0 ){ ++ rc = sqlite3CompressVFSModuleInit(); ++ if( rc!= SQLITE_OK ){ ++ *pzErrMsg = sqlite3_mprintf("load so of compressvfs error."); ++ rc = SQLITE_ERROR; ++ goto parse_uri_out; ++ } ++ } ++#endif ++ + *ppVfs = sqlite3_vfs_find(zVfs); + if( *ppVfs==0 ){ + *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs); +@@ -256147,9 +256532,9 @@ static int PragmaCksumPersistEnable(sqlite3 *db, int iDb, Parse *parse, const ch + } + return 1; + } +-extern sqlite3_file *sqlite3_get_orig_file(sqlite3_file *file); ++extern sqlite3_file *cksmvfsGetOrigFile(sqlite3_file *file); + #else +-static sqlite3_file *sqlite3_get_orig_file(sqlite3_file *file) { ++static sqlite3_file *cksmvfsGetOrigFile(sqlite3_file *file) { + return file; + } + #endif /* SQLITE_CKSUMVFS_STATIC */ +@@ -256157,13 +256542,22 @@ static sqlite3_file *sqlite3_get_orig_file(sqlite3_file *file) { + #if SQLITE_OS_UNIX + #define SQLITE_CHECK_FILE_ID_UNIX 1 + #define SQLITE_CHECK_FILE_ID_CKSM 2 ++#define SQLITE_CHECK_FILE_ID_COMPRESS 3 + +-// checkFileId should not be 0, it must be SQLITE_CHECK_FILE_ID_UNIX(1) or SQLITE_CHECK_FILE_ID_CKSM(2) ++// checkFileId should not be 0 ++// it must be SQLITE_CHECK_FILE_ID_UNIX(1) or SQLITE_CHECK_FILE_ID_CKSM(2) or SQLITE_CHECK_FILE_ID_COMPRESS(3) + static unixFile *Sqlite3GetUnixFile(sqlite3_file *file, u8 checkFileId) { +- if (checkFileId == SQLITE_CHECK_FILE_ID_UNIX) { ++ if( checkFileId == SQLITE_CHECK_FILE_ID_UNIX ){ + return (unixFile*)file; ++ }else if( checkFileId == SQLITE_CHECK_FILE_ID_CKSM ){ ++ return (unixFile*)cksmvfsGetOrigFile(file); ++ } ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ if( compressvfsGetOrigFilePtr==NULL ){ ++ return (unixFile*)compressvfsGetOrigFilePtr(file); + } +- return (unixFile*)sqlite3_get_orig_file(file); ++#endif ++ return (unixFile*)file; + } + #endif /* SQLITE_OS_UNIX */ + +@@ -256460,6 +256854,8 @@ static MetaDwrHdr *AllocInitMetaHeaderDwr(Pager *pPager) { + hdr->checkFileId = SQLITE_CHECK_FILE_ID_UNIX; + } else if (sqlite3_stricmp(pPager->pVfs->zName, "cksmvfs") == 0) { + hdr->checkFileId = SQLITE_CHECK_FILE_ID_CKSM; ++ } else if (sqlite3_stricmp(pPager->pVfs->zName, "compressvfs") == 0) { ++ hdr->checkFileId = SQLITE_CHECK_FILE_ID_COMPRESS; + } else { + hdr->checkFileId = 0; + } +@@ -257407,14 +257803,23 @@ static void DumpTrxProcessLocks(unixFile *file, char *dumpBuf, int dumpBufLen) + + static void DumpWalLocks(unixFile *file, u8 walEnabled, char *dumpBuf, int dumpBufLen) + { ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ if ((file->pShm == NULL || file->pShm->pShmNode == NULL) && ++ (file->pCompressShm == NULL || file->pCompressShm->pShmNode == NULL)) { ++#else + if (file->pShm == NULL || file->pShm->pShmNode == NULL) { ++#endif + sqlite3_log(SQLITE_ERROR, "[SQLite]Wal mode disabled! pShm or pShmNode is NULL"); + return; + } + if (!walEnabled) { + sqlite3_log(SQLITE_ERROR, "[SQLite] walEnabled false"); + } ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ unixShmNode *pShmNode = file->pShm->pShmNode == NULL ? file->pCompressShm->pShmNode : file->pShm->pShmNode; ++#else + unixShmNode *pShmNode = file->pShm->pShmNode; ++#endif + char *tmp = dumpBuf; + int availLen = dumpBufLen - 1; + dumpBuf[availLen] = '\0'; +@@ -258600,6 +259005,11 @@ struct sqlite3_api_routines_hw { + int (*is_support_binlog)(void); + int (*replay_binlog)(sqlite3*, sqlite3*); + int (*clean_binlog)(sqlite3*, BinlogFileCleanModeE); ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ int (*compressdb_backup)(sqlite3*, const char*); ++#else ++ void *dymmyFunc1; ++#endif + }; + + typedef struct sqlite3_api_routines_hw sqlite3_api_routines_hw; +@@ -258626,6 +259036,11 @@ static const sqlite3_api_routines_hw sqlite3HwApis = { + 0, + 0, + #endif/* SQLITE_ENABLE_BINLOG */ ++#ifdef SQLITE_ENABLE_PAGE_COMPRESS ++ sqlite3_compressdb_backup, ++#else ++ 0, ++#endif/* SQLITE_ENABLE_PAGE_COMPRESS */ + }; + + EXPORT_SYMBOLS const sqlite3_api_routines *sqlite3_export_symbols = &sqlite3Apis; +-- +2.47.0.windows.2 + diff --git a/patch/0011-Bug-fixes-on-current-version.patch b/patch/0012-Bugfix-on-current-version.patch similarity index 89% rename from patch/0011-Bug-fixes-on-current-version.patch rename to patch/0012-Bugfix-on-current-version.patch index 1c5aaf3..9b22340 100644 --- a/patch/0011-Bug-fixes-on-current-version.patch +++ b/patch/0012-Bugfix-on-current-version.patch @@ -1,18 +1,17 @@ -From 7bb232b30ce94a323c92c93a30df658a02d4d702 Mon Sep 17 00:00:00 2001 +From 409c2be5857723d06c50553a93e954dd1ec3b25b Mon Sep 17 00:00:00 2001 From: MartinChoo <214582617@qq.com> -Date: Sat, 7 Jun 2025 23:07:45 +0800 -Subject: [PATCH] Bug fixes on current version +Date: Thu, 3 Jul 2025 23:13:43 +0800 +Subject: [PATCH 12/12] Bugfix on current version -Signed-off-by: MartinChoo <214582617@qq.com> --- src/sqlite3.c | 239 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 193 insertions(+), 46 deletions(-) diff --git a/src/sqlite3.c b/src/sqlite3.c -index 80d7979..6ab872e 100644 +index 67a78e3..19b513b 100644 --- a/src/sqlite3.c +++ b/src/sqlite3.c -@@ -38786,8 +38786,8 @@ static void enableDbFileDelMonitor(int32_t fd) +@@ -38802,8 +38802,8 @@ static void enableDbFileDelMonitor(int32_t fd) } flags |= HMFS_MONITOR_FL; ret = ioctl(fd, HMFS_IOCTL_HW_SET_FLAGS, &flags); @@ -23,7 +22,7 @@ index 80d7979..6ab872e 100644 } } -@@ -57786,6 +57786,8 @@ static void MetaDwrCheckVacuum(BtShared *pBt); +@@ -57978,6 +57978,8 @@ static void MetaDwrCheckVacuum(BtShared *pBt); static int MetaDwrRecoverAndBeginTran(Btree *pBt, int wrflag, int *pSchemaVersion); static int MetaDwrOpenAndCheck(Btree *pBt); static void MetaDwrDisable(Btree *pBt); @@ -32,7 +31,7 @@ index 80d7979..6ab872e 100644 #define META_HEADER_CHANGED 1 #define META_SCHEMA_CHANGED 2 #define META_IN_RECOVERY 1 -@@ -59564,11 +59566,12 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ +@@ -59756,11 +59758,12 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); } #ifdef SQLITE_META_DWR @@ -50,7 +49,7 @@ index 80d7979..6ab872e 100644 } #endif if( pagerUseWal(pPager) ){ -@@ -74672,6 +74675,11 @@ static int lockBtree(BtShared *pBt){ +@@ -74865,6 +74868,11 @@ static int lockBtree(BtShared *pBt){ } if( nPage>nPageFile ){ if( sqlite3WritableSchema(pBt->db)==0 ){ @@ -62,7 +61,7 @@ index 80d7979..6ab872e 100644 rc = SQLITE_CORRUPT_BKPT; goto page1_init_failed; }else{ -@@ -121520,8 +121528,8 @@ static void attachFunc( +@@ -121725,8 +121733,8 @@ static void attachFunc( if( rc==SQLITE_OK ){ extern int sqlite3CodecAttach(sqlite3*, int, const void*, int); extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); @@ -73,7 +72,7 @@ index 80d7979..6ab872e 100644 int t = sqlite3_value_type(argv[2]); switch( t ){ case SQLITE_INTEGER: -@@ -121538,14 +121546,7 @@ static void attachFunc( +@@ -121743,14 +121751,7 @@ static void attachFunc( break; case SQLITE_NULL: @@ -89,7 +88,7 @@ index 80d7979..6ab872e 100644 break; } } -@@ -183717,10 +183718,12 @@ opendb_out: +@@ -184118,10 +184119,12 @@ opendb_out: db->eOpenState = SQLITE_STATE_SICK; } #ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK @@ -106,7 +105,7 @@ index 80d7979..6ab872e 100644 #endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ #ifdef SQLITE_ENABLE_BINLOG sqlite3BinlogReset(db); -@@ -204166,6 +204169,39 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ +@@ -204567,6 +204570,39 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ return rc; } @@ -146,7 +145,7 @@ index 80d7979..6ab872e 100644 /* ** Implementation of offsets() function. */ -@@ -204202,6 +204238,12 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( +@@ -204603,6 +204639,12 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( sCtx.iDocid = pCsr->iPrevId; sCtx.pCsr = pCsr; @@ -159,7 +158,7 @@ index 80d7979..6ab872e 100644 /* Loop through the table columns, appending offset information to ** string-buffer res for each column. */ -@@ -254388,7 +254430,20 @@ SQLITE_API int sqlite3_stmt_init( +@@ -254789,7 +254831,20 @@ SQLITE_API int sqlite3_stmt_init( /* Return the source-id for this library */ SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } /************************** End of sqlite3.c ******************************/ @@ -181,7 +180,7 @@ index 80d7979..6ab872e 100644 #ifdef SQLITE_HAS_CODEC /************** Begin file hw_codec_openssl.h *******************************/ #ifndef EXPOSE_INTERNAL_FUNC -@@ -255582,6 +255637,7 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *pKey, int nKey){ +@@ -255983,6 +256038,7 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *pKey, int nKey){ } } #endif @@ -189,7 +188,7 @@ index 80d7979..6ab872e 100644 #ifdef SQLITE_CODEC_ATTACH_CHANGED int rc = sqlite3CodecInitContext(ctx, p, pKey, nKey, nDb); #else -@@ -256176,7 +256232,12 @@ typedef struct MetaDwrHdr { +@@ -256586,7 +256642,12 @@ typedef struct MetaDwrHdr { u32 pageSz; u32 pageCnt; u64 dbFileInode; @@ -203,7 +202,7 @@ index 80d7979..6ab872e 100644 u32 checkSum; u8 *zones; Pgno *pages; -@@ -256539,6 +256600,87 @@ static inline u64 CaculateMetaDwrWriteOffset(int pageSz, u32 idx, u8 zone) { +@@ -256951,6 +257012,87 @@ static inline u64 CaculateMetaDwrWriteOffset(int pageSz, u32 idx, u8 zone) { return META_DWR_HEADER_PAGE_SIZE + pageSz * (idx * 2 + zone); } @@ -291,7 +290,7 @@ index 80d7979..6ab872e 100644 static void MetaDwrUpdateHeaderDbInfo(BtShared *pBt) { MetaDwrHdr *hdr = pBt->pPager->metaHdr; // 28 offset: dbSize, freelist pageNo, freelist pages count, schema cookie -@@ -256759,11 +256901,12 @@ static int MetaDwrOpenFile(Pager *pPager, u8 openCreate) { +@@ -257171,11 +257313,12 @@ static int MetaDwrOpenFile(Pager *pPager, u8 openCreate) { if (pPager->metaMapPage == NULL) { sqlite3_int64 sz = META_DWR_HEADER_PAGE_SIZE; sqlite3OsFileControlHint(metaFd, SQLITE_FCNTL_CHUNK_SIZE, &sz); @@ -309,7 +308,7 @@ index 80d7979..6ab872e 100644 } } #endif /* SQLITE_OS_UNIX */ -@@ -257137,7 +257280,7 @@ CHK_RESTORE_OUT: +@@ -257549,7 +257692,7 @@ CHK_RESTORE_OUT: return rc; } @@ -318,7 +317,7 @@ index 80d7979..6ab872e 100644 { #if SQLITE_OS_UNIX if (pPager->pVfs == NULL) { -@@ -257151,10 +257294,18 @@ static inline u8 IsConnectionValidForCheck(Pager *pPager) +@@ -257563,10 +257706,18 @@ static inline u8 IsConnectionValidForCheck(Pager *pPager) checkFileId = SQLITE_CHECK_FILE_ID_CKSM; } unixFile *fd = Sqlite3GetUnixFile(pPager->fd, checkFileId); @@ -338,7 +337,7 @@ index 80d7979..6ab872e 100644 return 1; #else return 0; -@@ -257164,7 +257315,7 @@ static inline u8 IsConnectionValidForCheck(Pager *pPager) +@@ -257576,7 +257727,7 @@ static inline u8 IsConnectionValidForCheck(Pager *pPager) static int MetaDwrOpenAndCheck(Btree *pBt) { Pager *pPager = pBt->pBt->pPager; @@ -347,7 +346,7 @@ index 80d7979..6ab872e 100644 return SQLITE_OK; } #ifdef SQLITE_HAS_CODEC -@@ -257209,7 +257360,7 @@ DWR_OPEN_OUT: +@@ -257621,7 +257772,7 @@ DWR_OPEN_OUT: static void MetaDwrDisable(Btree *pBt) { Pager *pPager = pBt->pBt->pPager; @@ -356,7 +355,7 @@ index 80d7979..6ab872e 100644 return; } #ifdef SQLITE_HAS_CODEC -@@ -257235,19 +257386,6 @@ static void MetaDwrDisable(Btree *pBt) +@@ -257647,19 +257798,6 @@ static void MetaDwrDisable(Btree *pBt) #endif /* SQLITE_META_DWR */ #if SQLITE_OS_UNIX @@ -376,7 +375,7 @@ index 80d7979..6ab872e 100644 static void ResetLockStatus(void) { (void)memset(&g_lockStatus, 0, sizeof(g_lockStatus)); -@@ -257351,8 +257489,13 @@ static inline const char *FlockToName(int l_type) +@@ -257763,8 +257901,13 @@ static inline const char *FlockToName(int l_type) static int DumpProcessLocks(int fd, struct flock *lock, const char *lockName, char *dumpBuf, int bufLen) { @@ -391,7 +390,7 @@ index 80d7979..6ab872e 100644 sqlite3_log(SQLITE_ERROR, "[SQLite]Get wal file lock ofs %u failed, errno: %d", lock->l_start, errno); return 0; } -@@ -258422,7 +258565,11 @@ struct sqlite3_api_routines_hw { +@@ -259004,7 +259147,11 @@ struct sqlite3_api_routines_hw { int (*rekey_v2)(sqlite3*,const char*,const void*,int); int (*is_support_binlog)(void); int (*replay_binlog)(sqlite3*, sqlite3*); @@ -400,9 +399,9 @@ index 80d7979..6ab872e 100644 +#else + void *dymmyFunc; +#endif - }; - - typedef struct sqlite3_api_routines_hw sqlite3_api_routines_hw; + #ifdef SQLITE_ENABLE_PAGE_COMPRESS + int (*compressdb_backup)(sqlite3*, const char*); + #else -- 2.47.0.windows.2 diff --git a/patch/BUILD.gn b/patch/BUILD.gn index 2abaadd..f3232ac 100644 --- a/patch/BUILD.gn +++ b/patch/BUILD.gn @@ -24,6 +24,7 @@ action("apply_patch") { outputs = [ "$sqlite_dst_dir/ext/misc/cksumvfs.c", "$sqlite_dst_dir/src/shell.c", + "$sqlite_dst_dir/src/compressvfs.c", "$sqlite_dst_dir/src/sqlite3.c", "$sqlite_dst_dir/src/sqlite3icu.c", "$sqlite_dst_dir/include/sqlite3ext.h", diff --git a/unittest/BUILD.gn b/unittest/BUILD.gn index d32df9e..6974b42 100644 --- a/unittest/BUILD.gn +++ b/unittest/BUILD.gn @@ -34,6 +34,8 @@ ohos_unittest("libsqlittest") { module_out_path = module_output_path sources = [ + "./common.cpp", + "./sqlite_compress_test.cpp", "./sqlite_test.cpp", ] diff --git a/unittest/common.cpp b/unittest/common.cpp new file mode 100644 index 0000000..9f98f94 --- /dev/null +++ b/unittest/common.cpp @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +namespace OHOS { +namespace SQLiteTest { +int Common::RemoveDir(const char *dir) +{ + if (dir == nullptr) { + return TEST_STATUS_ERR; + } + if (access(dir, F_OK) != 0) { + return TEST_STATUS_OK; + } + struct stat dirStat; + if (stat(dir, &dirStat) < 0) { + return TEST_STATUS_ERR; + } + DIR *dirPtr = nullptr; + struct dirent *dr = nullptr; + if (S_ISREG(dirStat.st_mode)) { + remove(dir); + } else if (S_ISDIR(dirStat.st_mode)) { + dirPtr = opendir(dir); + while ((dr = readdir(dirPtr)) != nullptr) { + if ((strcmp(".", dr->d_name) == 0) || (strcmp("..", dr->d_name) == 0)) { + continue; + } + string dirName = dir; + dirName += "/"; + dirName += dr->d_name; + RemoveDir(dirName.c_str()); + } + closedir(dirPtr); + rmdir(dir); + } else { + return TEST_STATUS_ERR; + } + return TEST_STATUS_OK; +} + +int Common::MakeDir(const char *dir) +{ + if (dir == nullptr) { + return TEST_STATUS_ERR; + } + if (strlen(dir) > PATH_MAX) { + return TEST_STATUS_ERR; + } + if (access(dir, 0) != -1) { + return TEST_STATUS_OK; + } + char tmpPath[PATH_MAX + 1] = {0}; + const char *pcur = dir; + int pos = 0; + while (*pcur++ != '\0') { + tmpPath[pos++] = *const_cast(pcur - 1); + if ((*pcur == '/' || *pcur == '\0') && access(tmpPath, 0) != 0 && strlen(tmpPath) > 0) { + if (mkdir(tmpPath, (S_IRUSR | S_IWUSR | S_IXUSR)) != 0) { + return TEST_STATUS_ERR; + } + } + } + return TEST_STATUS_OK; +} +} // namespace SQLiteTest +} // namespace OHOS \ No newline at end of file diff --git a/unittest/common.h b/unittest/common.h new file mode 100644 index 0000000..f10c192 --- /dev/null +++ b/unittest/common.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef COMMON_H +#define COMMON_H + +namespace OHOS { +namespace SQLiteTest { + +#define TEST_STATUS_OK 0 +#define TEST_STATUS_ERR (-1) + +class Common { +public: + static int RemoveDir(const char *dir); + static int MakeDir(const char *dir); +}; +} // namespace SQLiteTest +} // namespace OHOS + +#endif /* COMMON_H */ diff --git a/unittest/sqlite_compress_test.cpp b/unittest/sqlite_compress_test.cpp new file mode 100644 index 0000000..cb4bc2b --- /dev/null +++ b/unittest/sqlite_compress_test.cpp @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "sqlite3sym.h" + +using namespace testing::ext; +using namespace OHOS::SQLiteTest; + +#define TEST_DIR "./sqlitecompresstest" +#define TEST_DB (TEST_DIR "/test.db") +#define TEST_PRESET_TABLE_COUNT 20 +#define TEST_PRESET_DATA_COUNT 100 +#define TEST_COMPRESS_META_TABLE (2) + +namespace Test { +class SQLiteCompressTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); + + static void UtSqliteLogPrint(const void *data, int err, const char *msg); + static void UtCheckDb(const std::string &dbPath, int tableCount = TEST_COMPRESS_META_TABLE); + static void UtBackupDatabase(sqlite3 *srcDb, sqlite3 *destDb); + static void UtBackupCompressDatabase(sqlite3 *srcDb, sqlite3 *destDb); + + static sqlite3 *db; + static const std::string &UT_DDL_CREATE_DEMO; + static const std::string &UT_DML_INSERT_DEMO; + static const std::string &UT_SQL_SELECT_META; + static const std::string &UT_DDL_CREATE_PHONE; + static const std::string &UT_PHONE_DESC; +}; + +sqlite3 *SQLiteCompressTest::db = nullptr; +const std::string &SQLiteCompressTest::UT_DDL_CREATE_DEMO = "CREATE TABLE demo(id INTEGER PRIMARY KEY, name TEXT);"; +const std::string &SQLiteCompressTest::UT_DML_INSERT_DEMO = "INSERT INTO demo(id, name) VALUES(110, 'Call 110!');"; +const std::string &SQLiteCompressTest::UT_SQL_SELECT_META = "SELECT COUNT(1) FROM sqlite_master;"; +const std::string &SQLiteCompressTest::UT_DDL_CREATE_PHONE = + "CREATE TABLE IF NOT EXISTS phone(id INTEGER PRIMARY KEY, name TEXT, brand TEXT, price REAL, type int, desc TEXT);"; +const std::string &SQLiteCompressTest::UT_PHONE_DESC = "The phone is easy to use and popular. 優點(繁体)系统(简体)稳定"; + +void SQLiteCompressTest::UtSqliteLogPrint(const void *data, int err, const char *msg) +{ + std::cout << "SQLiteCompressTest xLog err:" << err << ", msg:" << msg << std::endl; +} + +void SQLiteCompressTest::UtCheckDb(const std::string &dbPath, int tableCount) +{ + sqlite3 *tmpDb = nullptr; + EXPECT_EQ(sqlite3_open_v2(dbPath.c_str(), &tmpDb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr), + SQLITE_OK) << ", compress db:" << dbPath.c_str(); + sqlite3_stmt *sqlStmt = nullptr; + EXPECT_EQ(sqlite3_prepare_v2(tmpDb, UT_SQL_SELECT_META.c_str(), -1, &sqlStmt, nullptr), + SQLITE_OK) << ", compress db:" << dbPath.c_str(); + EXPECT_EQ(sqlite3_step(sqlStmt), SQLITE_ROW) << ", compress db:" << dbPath.c_str(); + int count = sqlite3_column_int(sqlStmt, 0); + EXPECT_EQ(count, tableCount) << ", compress db:" << dbPath.c_str(); + sqlite3_finalize(sqlStmt); + sqlite3_close_v2(tmpDb); +} + +void SQLiteCompressTest::UtBackupDatabase(sqlite3 *srcDb, sqlite3 *destDb) +{ + sqlite3_backup *back = nullptr; + back = sqlite3_backup_init(destDb, "main", srcDb, "main"); + EXPECT_TRUE(back != nullptr); + EXPECT_EQ(sqlite3_backup_step(back, -1), SQLITE_DONE); + sqlite3_backup_finish(back); +} + +void SQLiteCompressTest::SetUpTestCase(void) +{ + Common::RemoveDir(TEST_DIR); + Common::MakeDir(TEST_DIR); +} + +void SQLiteCompressTest::TearDownTestCase(void) +{ +} + +void SQLiteCompressTest::SetUp(void) +{ + std::string command = "rm -rf "; + command += TEST_DIR "/*"; + system(command.c_str()); + sqlite3_config(SQLITE_CONFIG_LOG, &SQLiteCompressTest::UtSqliteLogPrint, NULL); + EXPECT_EQ(sqlite3_open(TEST_DB, &db), SQLITE_OK); + EXPECT_EQ(sqlite3_exec(db, UT_DDL_CREATE_PHONE.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + for (int i = 0; i < TEST_PRESET_TABLE_COUNT; i++) { + std::string ddl = "CREATE TABLE IF NOT EXISTS test"; + ddl += std::to_string(i + 1); + ddl += "(id INTEGER PRIMARY KEY, field1 INTEGER, field2 REAL, field3 TEXT, field4 BLOB, field5 TEXT);"; + EXPECT_EQ(sqlite3_exec(db, ddl.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + } + std::vector phoneList = {"Huawei", "Samsung", "Apple", "Xiaomi", "Oppo", "Vivo", "Realme"}; + for (int i = 0; i < TEST_PRESET_DATA_COUNT; i++) { + std::string dml = "INSERT INTO phone(id, name, brand, price, type, desc) VALUES("; + dml += std::to_string(i + 1) + ",'"; + dml += std::to_string(i + 1) + " Martin''s Phone','"; + dml += std::to_string(i + 1) + phoneList[i%phoneList.size()] + "',"; + dml += std::to_string(i + 1.0) + ","; + dml += std::to_string(i + 1) + ",'" + UT_PHONE_DESC + UT_PHONE_DESC + UT_PHONE_DESC + UT_PHONE_DESC + "');"; + EXPECT_EQ(sqlite3_exec(db, dml.c_str(), nullptr, nullptr, nullptr), SQLITE_OK) << sqlite3_errmsg(db); + } + sqlite3_close(db); + EXPECT_EQ(sqlite3_open(TEST_DB, &db), SQLITE_OK); +} + +void SQLiteCompressTest::TearDown(void) +{ + sqlite3_close(db); + db = nullptr; + sqlite3_config(SQLITE_CONFIG_LOG, NULL, NULL); +} + +/** + * @tc.name: CompressTest001 + * @tc.desc: Test to enable page compression on brand new db. + * @tc.type: FUNC + */ +HWTEST_F(SQLiteCompressTest, CompressTest001, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create a brand new db while page compression enabled with sqlite3_open_v2 + * @tc.expected: step1. Execute successfully + */ + std::string dbPath1 = TEST_DIR "/compresstest0010.db"; + sqlite3 *compDb = nullptr; + EXPECT_EQ(sqlite3_open_v2(dbPath1.c_str(), &compDb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, "compressvfs"), + SQLITE_OK); + EXPECT_EQ(sqlite3_exec(compDb, UT_DDL_CREATE_DEMO.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + EXPECT_EQ(sqlite3_exec(compDb, UT_DML_INSERT_DEMO.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + sqlite3_close_v2(compDb); + compDb = nullptr; + UtCheckDb(dbPath1); + /** + * @tc.steps: step2. Create a brand new db while page compression enabled through uri + * @tc.expected: step2. Execute successfully + */ + std::string dbFileUri = "file:"; + dbFileUri += TEST_DIR "/compresstest0011.db"; + dbFileUri += "?vfs=compressvfs"; + EXPECT_EQ(sqlite3_open(dbFileUri.c_str(), &compDb), SQLITE_OK); + EXPECT_EQ(sqlite3_exec(compDb, UT_DDL_CREATE_DEMO.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + EXPECT_EQ(sqlite3_exec(compDb, UT_DML_INSERT_DEMO.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + sqlite3_close(compDb); + compDb = nullptr; + UtCheckDb(TEST_DIR "/compresstest0011.db"); +} + +/** + * @tc.name: CompressTest002 + * @tc.desc: Test to try enable page compression on none compress db + * @tc.type: FUNC + */ +HWTEST_F(SQLiteCompressTest, CompressTest002, TestSize.Level0) +{ + sqlite3 *compDb = nullptr; + EXPECT_EQ(sqlite3_open_v2(TEST_DB, &compDb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, "compressvfs"), + SQLITE_WARNING); + EXPECT_EQ(sqlite3_extended_errcode(compDb), SQLITE_WARNING_NOTCOMPRESSDB); + sqlite3_close_v2(compDb); + + EXPECT_EQ(sqlite3_open_v2(TEST_DB, &compDb, SQLITE_OPEN_READONLY, "compressvfs"), SQLITE_WARNING); + EXPECT_EQ(sqlite3_extended_errcode(compDb), SQLITE_WARNING_NOTCOMPRESSDB); + sqlite3_close_v2(compDb); + + std::string dbFileUri = "file:"; + dbFileUri += TEST_DB; + dbFileUri += "?vfs=compressvfs"; + EXPECT_EQ(sqlite3_open(dbFileUri.c_str(), &compDb), SQLITE_WARNING); + EXPECT_EQ(sqlite3_extended_errcode(compDb), SQLITE_WARNING_NOTCOMPRESSDB); + sqlite3_close_v2(compDb); +} + +/** + * @tc.name: CompressTest003 + * @tc.desc: Test to backup db and enable page compression on dest db + * @tc.type: FUNC + */ +HWTEST_F(SQLiteCompressTest, CompressTest003, TestSize.Level0) +{ + /** + * @tc.steps: step1. Compress db using Backup function:sqlite3_backup_step + * @tc.expected: step1. Execute successfully + */ + std::string dbPath1 = TEST_DIR "/compresstest0030.db"; + sqlite3 *compDb = nullptr; + EXPECT_EQ(sqlite3_open_v2(dbPath1.c_str(), &compDb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, "compressvfs"), + SQLITE_OK); + UtBackupDatabase(db, compDb); + sqlite3_close_v2(compDb); + UtCheckDb(dbPath1); + + /** + * @tc.steps: step2. Compress db using new Backup function:sqlite3_compressdb_backup + * @tc.expected: step2. Execute successfully + */ + EXPECT_EQ(sqlite3_open_v2(dbPath1.c_str(), &compDb, SQLITE_OPEN_READWRITE, nullptr), SQLITE_OK); + std::string dbPath2 = TEST_DIR "/restore0031.db"; + EXPECT_EQ(sqlite3_compressdb_backup(compDb, dbPath2.c_str()), SQLITE_DONE); + sqlite3_close_v2(compDb); + UtCheckDb(dbPath2, TEST_PRESET_TABLE_COUNT + 1); +} + +} // namespace Test diff --git a/unittest/sqlite_test.cpp b/unittest/sqlite_test.cpp index 71f6e10..9110525 100644 --- a/unittest/sqlite_test.cpp +++ b/unittest/sqlite_test.cpp @@ -43,7 +43,10 @@ static void UtPresetDb(const char *dbFile) * @tc.expected: step1. Execute successfully */ sqlite3 *db = NULL; - EXPECT_EQ(sqlite3_open(dbFile, &db), SQLITE_OK); + std::string dbFileUri = "file:"; + dbFileUri += dbFile; + dbFileUri += "?vfs=cksmvfs"; + EXPECT_EQ(sqlite3_open(dbFileUri.c_str(), &db), SQLITE_OK); /** * @tc.steps: step1. Enable cksumvfs using PRAGMA checksum_persist_enable, * @tc.expected: step1. Execute successfully @@ -141,7 +144,10 @@ HWTEST_F(LibSQLiteTest, Lib_SQLite_Test_001, TestSize.Level0) */ sqlite3 *db = NULL; UtCorruptDb(TEST_DB, 3 * 4096 + 1000, 1, 0xFF); // 3 * 4096 + 1000 is the target page's position in file - EXPECT_EQ(sqlite3_open(TEST_DB, &db), SQLITE_OK); + std::string dbFileUri = "file:"; + dbFileUri += TEST_DB; + dbFileUri += "?vfs=cksmvfs"; + EXPECT_EQ(sqlite3_open(dbFileUri.c_str(), &db), SQLITE_OK); static const char *UT_SQL_SELECT_TABLE = "SELECT COUNT(*) FROM salary WHERE entryId=3;"; EXPECT_EQ(sqlite3_exec(db, UT_SQL_SELECT_TABLE, NULL, NULL, NULL), SQLITE_IOERR); /** @@ -174,7 +180,10 @@ HWTEST_F(LibSQLiteTest, Lib_SQLite_Test_002, TestSize.Level0) */ sqlite3 *db = NULL; UtCorruptDb(TEST_DB, 3 * 4096 + 1000, 1, 0xFF); // 3 * 4096 + 1000 is the target page's position in file - EXPECT_EQ(sqlite3_open(TEST_DB, &db), SQLITE_OK); + std::string dbFileUri = "file:"; + dbFileUri += TEST_DB; + dbFileUri += "?vfs=cksmvfs"; + EXPECT_EQ(sqlite3_open(dbFileUri.c_str(), &db), SQLITE_OK); static const char *UT_SQL_SELECT_TABLE_1 = "SELECT COUNT(*) FROM salary WHERE entryId=3;"; EXPECT_EQ(sqlite3_exec(db, UT_SQL_SELECT_TABLE_1, NULL, NULL, NULL), SQLITE_IOERR); /** -- Gitee